2025-11-07 02:30:02 +00:00
|
|
|
import io
|
|
|
|
|
import logging
|
|
|
|
|
|
|
|
|
|
from markitdown import MarkItDown
|
|
|
|
|
|
2026-05-07 08:05:21 +00:00
|
|
|
from docreader.config import CONFIG
|
2025-11-07 02:30:02 +00:00
|
|
|
from docreader.models.document import Document
|
|
|
|
|
from docreader.parser.base_parser import BaseParser
|
|
|
|
|
from docreader.parser.chain_parser import PipelineParser
|
2026-05-07 08:05:21 +00:00
|
|
|
from docreader.parser.concurrency import parser_worker_limit
|
2026-08-12 02:59:49 +00:00
|
|
|
from docreader.parser.docx_merge import fill_vertical_merged_cells_docx
|
2025-11-07 02:30:02 +00:00
|
|
|
from docreader.parser.markdown_parser import MarkdownParser
|
2026-06-03 04:00:09 +00:00
|
|
|
from docreader.parser.ppt_convert import normalize_ppt_bytes
|
|
|
|
|
from docreader.parser.pptx_media import (
|
|
|
|
|
attach_pptx_media_to_markdown,
|
|
|
|
|
markdown_needs_pptx_media_attach,
|
|
|
|
|
)
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
from docreader.parser.xlsx_repair import (
|
|
|
|
|
sanitize_xlsx_styles,
|
|
|
|
|
strip_unreadable_ranges_xlsx,
|
|
|
|
|
)
|
2025-11-07 02:30:02 +00:00
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class StdMarkitdownParser(BaseParser):
|
|
|
|
|
"""
|
2026-01-16 03:33:58 +00:00
|
|
|
Standard MarkItDown Parser Wrapper
|
2025-11-07 02:30:02 +00:00
|
|
|
|
2026-01-16 03:33:58 +00:00
|
|
|
This parser uses the markitdown library to convert various document formats
|
|
|
|
|
(docx, pptx, pdf, etc.) into text/markdown.
|
2025-11-07 02:30:02 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, *args, **kwargs):
|
2026-01-16 03:33:58 +00:00
|
|
|
# 这里的 super() 会调用 BaseParser 的初始化,确保 self.file_type 被正确赋值
|
|
|
|
|
super().__init__(*args, **kwargs)
|
2025-11-07 02:30:02 +00:00
|
|
|
self.markitdown = MarkItDown()
|
|
|
|
|
|
2026-01-16 03:33:58 +00:00
|
|
|
def parse_into_text(self, content: bytes) -> Document:
|
2026-01-15 09:46:20 +00:00
|
|
|
"""
|
2026-01-16 03:33:58 +00:00
|
|
|
Parses content using MarkItDown.
|
|
|
|
|
Uses self.file_type (inherited from BaseParser) to hint the stream format.
|
2026-01-15 09:46:20 +00:00
|
|
|
"""
|
2026-01-16 03:33:58 +00:00
|
|
|
ext = self.file_type
|
2026-06-03 04:00:09 +00:00
|
|
|
ft = (ext or "").lstrip(".").lower()
|
|
|
|
|
pptx_bytes: bytes | None = None
|
|
|
|
|
if ft in ("ppt", "pptx"):
|
|
|
|
|
content, ext = normalize_ppt_bytes(content, ft)
|
|
|
|
|
pptx_bytes = content
|
|
|
|
|
ft = "pptx"
|
2026-08-12 02:59:49 +00:00
|
|
|
elif ft == "docx":
|
|
|
|
|
content = fill_vertical_merged_cells_docx(content)
|
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638)
* fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads
openpyxl raises TypeError('expected ... Fill') on <fill> elements with no
patternFill/gradientFill child — cloud-doc exporters, LabView and Acumatica
ERP emit these, upstream treats the workbooks as non-conformant, and both
parse engines died on files Excel opens fine (#3637).
Extend the zip-level repair module with sanitize_xlsx_styles: scan
xl/styles.xml, replace each malformed fill in place with a no-op
patternFill (index-preserving, so cellXf fillId references stay valid),
rewrite the zip only when something needed fixing. The builtin chain
retries the merge-cell prefill once on sanitized bytes; the markitdown
chain sanitizes xlsx/xlsm before MarkItDown sees them. Fixes #3637.
* fix(docreader): log sanitized fill count and never raise from the sanitizer
Fold review feedback into the styles sanitizer: a structured log line on
every repair (count included, doubling as the recurrence meter for #3637,
mirroring the sharedStrings repair log) and a never-raise wrapper so an
internal sanitizer failure passes the original bytes through instead of
trading the caller's error for its own.
* fix(docreader): read the workbook stylesheet by its exact part name
The styles sanitizer picked its part with a suffix match over a set of
zip member names, so a workbook that also carries
xl/richData/richStyles.xml (Excel 365 in-cell images / data types) got
the wrong part on some PYTHONHASHSEED values and skipped the repair.
Read xl/styles.xml, the one stylesheet openpyxl loads.
---------
Co-authored-by: vicfei <vicfei@users.noreply.github.com>
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:23:57 +00:00
|
|
|
elif ft in ("xlsx", "xlsm"):
|
|
|
|
|
# MarkItDown reads spreadsheets through pandas/openpyxl, so the
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
# ranges openpyxl cannot parse (#3599) and the non-conforming
|
|
|
|
|
# styles.xml fills (#3637) that break the builtin ExcelParser break
|
|
|
|
|
# this engine too. Strip the ranges first: the fill repair alone
|
|
|
|
|
# would still leave the range TypeError. Both passes return None
|
|
|
|
|
# for clean workbooks.
|
|
|
|
|
readable = strip_unreadable_ranges_xlsx(content)
|
|
|
|
|
if readable is not None:
|
|
|
|
|
content = readable
|
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638)
* fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads
openpyxl raises TypeError('expected ... Fill') on <fill> elements with no
patternFill/gradientFill child — cloud-doc exporters, LabView and Acumatica
ERP emit these, upstream treats the workbooks as non-conformant, and both
parse engines died on files Excel opens fine (#3637).
Extend the zip-level repair module with sanitize_xlsx_styles: scan
xl/styles.xml, replace each malformed fill in place with a no-op
patternFill (index-preserving, so cellXf fillId references stay valid),
rewrite the zip only when something needed fixing. The builtin chain
retries the merge-cell prefill once on sanitized bytes; the markitdown
chain sanitizes xlsx/xlsm before MarkItDown sees them. Fixes #3637.
* fix(docreader): log sanitized fill count and never raise from the sanitizer
Fold review feedback into the styles sanitizer: a structured log line on
every repair (count included, doubling as the recurrence meter for #3637,
mirroring the sharedStrings repair log) and a never-raise wrapper so an
internal sanitizer failure passes the original bytes through instead of
trading the caller's error for its own.
* fix(docreader): read the workbook stylesheet by its exact part name
The styles sanitizer picked its part with a suffix match over a set of
zip member names, so a workbook that also carries
xl/richData/richStyles.xml (Excel 365 in-cell images / data types) got
the wrong part on some PYTHONHASHSEED values and skipped the repair.
Read xl/styles.xml, the one stylesheet openpyxl loads.
---------
Co-authored-by: vicfei <vicfei@users.noreply.github.com>
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:23:57 +00:00
|
|
|
sanitized = sanitize_xlsx_styles(content)
|
|
|
|
|
if sanitized is not None:
|
|
|
|
|
content = sanitized
|
2026-08-12 02:59:49 +00:00
|
|
|
if ext and not ext.startswith("."):
|
2026-06-03 04:00:09 +00:00
|
|
|
ext = "." + ext
|
2026-01-16 03:33:58 +00:00
|
|
|
|
2026-05-07 08:05:21 +00:00
|
|
|
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
|
2026-06-03 04:00:09 +00:00
|
|
|
result = self._convert_markitdown(content, ext, keep_data_uris=True)
|
|
|
|
|
if result is None:
|
|
|
|
|
logger.warning(
|
|
|
|
|
"MarkItDown failed with embedded images for %s; retrying without data URIs",
|
|
|
|
|
ft or ext,
|
|
|
|
|
)
|
|
|
|
|
result = self._convert_markitdown(content, ext, keep_data_uris=False)
|
|
|
|
|
|
|
|
|
|
text = result.text_content
|
|
|
|
|
images: dict[str, str] = {}
|
|
|
|
|
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
|
|
|
|
|
text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
|
|
|
|
|
return Document(content=text, images=images)
|
|
|
|
|
|
2026-08-12 02:59:49 +00:00
|
|
|
def _convert_markitdown(
|
|
|
|
|
self, content: bytes, ext: str | None, *, keep_data_uris: bool
|
|
|
|
|
):
|
2026-06-03 04:00:09 +00:00
|
|
|
try:
|
|
|
|
|
return self.markitdown.convert(
|
2026-05-07 08:05:21 +00:00
|
|
|
io.BytesIO(content),
|
|
|
|
|
file_extension=ext,
|
2026-06-03 04:00:09 +00:00
|
|
|
keep_data_uris=keep_data_uris,
|
2026-05-07 08:05:21 +00:00
|
|
|
)
|
2026-06-03 04:00:09 +00:00
|
|
|
except Exception:
|
|
|
|
|
if keep_data_uris:
|
|
|
|
|
return None
|
|
|
|
|
raise
|
2025-11-07 02:30:02 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
class MarkitdownParser(PipelineParser):
|
2026-05-07 08:05:21 +00:00
|
|
|
_parser_cls = (StdMarkitdownParser, MarkdownParser)
|