WeKnora/docreader/parser/markitdown_parser.py

93 lines
3.4 KiB
Python
Raw Normal View History

import io
import logging
from markitdown import MarkItDown
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.docx_merge import fill_vertical_merged_cells_docx
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.ppt_convert import normalize_ppt_bytes
from docreader.parser.pptx_media import (
attach_pptx_media_to_markdown,
markdown_needs_pptx_media_attach,
)
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638) * fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads openpyxl raises TypeError('expected ... Fill') on <fill> elements with no patternFill/gradientFill child — cloud-doc exporters, LabView and Acumatica ERP emit these, upstream treats the workbooks as non-conformant, and both parse engines died on files Excel opens fine (#3637). Extend the zip-level repair module with sanitize_xlsx_styles: scan xl/styles.xml, replace each malformed fill in place with a no-op patternFill (index-preserving, so cellXf fillId references stay valid), rewrite the zip only when something needed fixing. The builtin chain retries the merge-cell prefill once on sanitized bytes; the markitdown chain sanitizes xlsx/xlsm before MarkItDown sees them. Fixes #3637. * fix(docreader): log sanitized fill count and never raise from the sanitizer Fold review feedback into the styles sanitizer: a structured log line on every repair (count included, doubling as the recurrence meter for #3637, mirroring the sharedStrings repair log) and a never-raise wrapper so an internal sanitizer failure passes the original bytes through instead of trading the caller's error for its own. * fix(docreader): read the workbook stylesheet by its exact part name The styles sanitizer picked its part with a suffix match over a set of zip member names, so a workbook that also carries xl/richData/richStyles.xml (Excel 365 in-cell images / data types) got the wrong part on some PYTHONHASHSEED values and skipped the repair. Read xl/styles.xml, the one stylesheet openpyxl loads. --------- Co-authored-by: vicfei <vicfei@users.noreply.github.com> Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:23:57 +00:00
from docreader.parser.xlsx_repair import sanitize_xlsx_styles
logger = logging.getLogger(__name__)
class StdMarkitdownParser(BaseParser):
"""
Standard MarkItDown Parser Wrapper
This parser uses the markitdown library to convert various document formats
(docx, pptx, pdf, etc.) into text/markdown.
"""
def __init__(self, *args, **kwargs):
# 这里的 super() 会调用 BaseParser 的初始化,确保 self.file_type 被正确赋值
super().__init__(*args, **kwargs)
self.markitdown = MarkItDown()
def parse_into_text(self, content: bytes) -> Document:
"""
Parses content using MarkItDown.
Uses self.file_type (inherited from BaseParser) to hint the stream format.
"""
ext = self.file_type
ft = (ext or "").lstrip(".").lower()
pptx_bytes: bytes | None = None
if ft in ("ppt", "pptx"):
content, ext = normalize_ppt_bytes(content, ft)
pptx_bytes = content
ft = "pptx"
elif ft == "docx":
content = fill_vertical_merged_cells_docx(content)
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638) * fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads openpyxl raises TypeError('expected ... Fill') on <fill> elements with no patternFill/gradientFill child — cloud-doc exporters, LabView and Acumatica ERP emit these, upstream treats the workbooks as non-conformant, and both parse engines died on files Excel opens fine (#3637). Extend the zip-level repair module with sanitize_xlsx_styles: scan xl/styles.xml, replace each malformed fill in place with a no-op patternFill (index-preserving, so cellXf fillId references stay valid), rewrite the zip only when something needed fixing. The builtin chain retries the merge-cell prefill once on sanitized bytes; the markitdown chain sanitizes xlsx/xlsm before MarkItDown sees them. Fixes #3637. * fix(docreader): log sanitized fill count and never raise from the sanitizer Fold review feedback into the styles sanitizer: a structured log line on every repair (count included, doubling as the recurrence meter for #3637, mirroring the sharedStrings repair log) and a never-raise wrapper so an internal sanitizer failure passes the original bytes through instead of trading the caller's error for its own. * fix(docreader): read the workbook stylesheet by its exact part name The styles sanitizer picked its part with a suffix match over a set of zip member names, so a workbook that also carries xl/richData/richStyles.xml (Excel 365 in-cell images / data types) got the wrong part on some PYTHONHASHSEED values and skipped the repair. Read xl/styles.xml, the one stylesheet openpyxl loads. --------- Co-authored-by: vicfei <vicfei@users.noreply.github.com> Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:23:57 +00:00
elif ft in ("xlsx", "xlsm"):
# MarkItDown reads spreadsheets through pandas/openpyxl, so the
# same non-conforming styles.xml fills that break the builtin
# ExcelParser break this engine too (#3637). The sanitize pass
# scans only styles.xml and returns None for clean workbooks.
sanitized = sanitize_xlsx_styles(content)
if sanitized is not None:
content = sanitized
if ext and not ext.startswith("."):
ext = "." + ext
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
result = self._convert_markitdown(content, ext, keep_data_uris=True)
if result is None:
logger.warning(
"MarkItDown failed with embedded images for %s; retrying without data URIs",
ft or ext,
)
result = self._convert_markitdown(content, ext, keep_data_uris=False)
text = result.text_content
images: dict[str, str] = {}
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
return Document(content=text, images=images)
def _convert_markitdown(
self, content: bytes, ext: str | None, *, keep_data_uris: bool
):
try:
return self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=keep_data_uris,
)
except Exception:
if keep_data_uris:
return None
raise
class MarkitdownParser(PipelineParser):
_parser_cls = (StdMarkitdownParser, MarkdownParser)