2025-12-01 09:43:26 +00:00
|
|
|
"""
|
|
|
|
|
Excel Parser Module
|
|
|
|
|
|
|
|
|
|
This module provides functionality to parse Excel files (.xlsx, .xls) into
|
|
|
|
|
structured Document objects with text content and chunks. It supports multiple
|
|
|
|
|
sheets and handles various Excel formats using pandas.
|
|
|
|
|
"""
|
2025-11-19 09:27:17 +00:00
|
|
|
import logging
|
2026-06-24 03:40:46 +00:00
|
|
|
import re
|
2025-11-19 09:27:17 +00:00
|
|
|
from io import BytesIO
|
2026-07-25 20:00:41 +00:00
|
|
|
from typing import Any, List
|
2025-11-19 09:27:17 +00:00
|
|
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
|
|
|
|
from docreader.models.document import Chunk, Document
|
|
|
|
|
from docreader.parser.base_parser import BaseParser
|
2026-06-03 04:00:09 +00:00
|
|
|
from docreader.parser.excel_convert import (
|
|
|
|
|
convert_excel_to_xlsx_bytes,
|
|
|
|
|
detect_excel_format,
|
|
|
|
|
engine_for_format,
|
|
|
|
|
normalize_excel_bytes,
|
|
|
|
|
)
|
|
|
|
|
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
from docreader.parser.xlsx_repair import (
|
|
|
|
|
repair_xlsx_bytes,
|
|
|
|
|
sanitize_xlsx_styles,
|
|
|
|
|
strip_unreadable_ranges_xlsx,
|
|
|
|
|
)
|
2025-11-19 09:27:17 +00:00
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
2026-06-24 03:40:46 +00:00
|
|
|
# Pattern to detect Excel image function strings that should be excluded from
|
|
|
|
|
# parsed text content. WPS uses =DISPIMG("ID",mode) to embed images in cells;
|
|
|
|
|
# when opened by other tools the formula may appear as plain text prefixed with
|
|
|
|
|
# "_xlfn." or "=". Office 365 uses =_xlfn.IMAGE(url, ...) similarly.
|
|
|
|
|
# The _xlfn. prefix is optional — WPS may omit it (e.g. =DISPIMG("ID",1)).
|
|
|
|
|
_IMAGE_FUNC_RE = re.compile(
|
|
|
|
|
r"^=?(_xlfn\.)?(DISPIMG|IMAGE)\(", re.IGNORECASE
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_image_function(value: object) -> bool:
|
|
|
|
|
"""Return True if *value* looks like an embedded-image function string."""
|
|
|
|
|
if not isinstance(value, str):
|
|
|
|
|
return False
|
|
|
|
|
return _IMAGE_FUNC_RE.match(value) is not None
|
|
|
|
|
|
2025-11-19 09:27:17 +00:00
|
|
|
|
|
|
|
|
class ExcelParser(BaseParser):
|
2025-12-01 09:43:26 +00:00
|
|
|
"""Parser for Excel files (.xlsx, .xls).
|
|
|
|
|
|
|
|
|
|
This parser extracts text content from Excel files by processing all sheets
|
|
|
|
|
and converting each row into a structured text format. Each row becomes a
|
|
|
|
|
separate chunk with key-value pairs.
|
|
|
|
|
|
|
|
|
|
Features:
|
|
|
|
|
- Supports multiple sheets in a single Excel file
|
|
|
|
|
- Automatically removes completely empty rows
|
|
|
|
|
- Converts each row to "column: value" format
|
|
|
|
|
- Creates individual chunks for each row for better granularity
|
|
|
|
|
|
|
|
|
|
Example:
|
|
|
|
|
>>> parser = ExcelParser()
|
|
|
|
|
>>> with open("data.xlsx", "rb") as f:
|
|
|
|
|
... content = f.read()
|
|
|
|
|
... document = parser.parse_into_text(content)
|
|
|
|
|
>>> print(document.content)
|
|
|
|
|
Name: John,Age: 30,City: NYC
|
|
|
|
|
Name: Jane,Age: 25,City: LA
|
|
|
|
|
"""
|
2026-07-25 20:00:41 +00:00
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
file_name: str = "",
|
|
|
|
|
file_type: str | None = None,
|
|
|
|
|
xlsx_first_row_as_header: Any = False,
|
|
|
|
|
**kwargs: Any,
|
|
|
|
|
):
|
|
|
|
|
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
|
|
|
|
|
self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
|
2025-12-01 09:43:26 +00:00
|
|
|
|
2025-11-19 09:27:17 +00:00
|
|
|
def parse_into_text(self, content: bytes) -> Document:
|
2025-12-01 09:43:26 +00:00
|
|
|
"""Parse Excel file bytes into a Document object.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
content: Raw bytes of the Excel file
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
Document: Parsed document containing:
|
|
|
|
|
- content: Full text with all rows from all sheets
|
|
|
|
|
- chunks: List of Chunk objects, one per row
|
|
|
|
|
|
|
|
|
|
Note:
|
|
|
|
|
- Empty rows (all NaN values) are automatically skipped
|
|
|
|
|
- Each row is formatted as: "col1: val1,col2: val2,..."
|
|
|
|
|
- Chunks maintain sequential ordering across all sheets
|
|
|
|
|
"""
|
2025-11-19 09:27:17 +00:00
|
|
|
chunks: List[Chunk] = []
|
|
|
|
|
text: List[str] = []
|
feat(chat): open cited documents at the cited passage (#3700)
* feat(chat): open cited documents at the cited passage
Clicking a citation used to show only the parsed chunk. It now opens the
original file in the references panel, jumps to the cited place and
highlights it, narrowed to the sentence the citation supports.
Parsing records where each chunk sits in the original file
(chunks.source_locators): PDF page and paragraph boxes from the docreader,
MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and
EPUB chapters aligned in Go against any parser engine's Markdown; text
offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites
are followed with a line-diff remapper. Scanned PDF pages get their OCR
text as quotes at search time so a sentence can pick its page.
The frontend renders PDFs with pdf.js and every other previewable type
with the existing preview, then highlights with the CSS Custom Highlight
API in a highlighter yellow that stays visible against the brand green.
* fix(docreader): serialize source blocks outside main so tests import it
test_docx_tables stubs docreader.parser in sys.modules, so importing
docreader.main from a later test failed in the full unittest discover run.
The wire conversion now lives in docreader.source_wire, which needs only
the proto.
* fix(chat): tighten source locators from review
- Remap rewritten-line-for-line stretches by line so scanned PDF pages
no longer drift onto the wrong page.
- Keep the source preview mounted for citations in the file already open.
- Wait for in-flight PDF page renders before looking for the text layer.
- Count image rune offsets in one pass; stream xlsx rows up to maxUnits.
- Start text locators after a UTF-8 BOM, as viewers decode without it.
- Only re-query image children for scanned-page results.
- Seek audio citations without autoplay and drop stale seek listeners.
- Fix the SQLite migration's cross-reference to versioned/000113.
* fix(migrations): renumber chunk source locators after main's chunk_images
main took versioned 000113 and sqlite 000032 for chunk_images, so the
source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
|
|
|
source_blocks: List[dict] = []
|
2025-11-19 09:27:17 +00:00
|
|
|
start, end = 0, 0
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
excel_file = _open_excel_file(content, file_type=self.file_type)
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Process each sheet in the Excel file
|
2025-11-19 09:27:17 +00:00
|
|
|
for excel_sheet_name in excel_file.sheet_names:
|
2026-07-25 20:00:41 +00:00
|
|
|
df = _read_sheet_dataframe(
|
|
|
|
|
excel_file,
|
|
|
|
|
excel_sheet_name,
|
|
|
|
|
xlsx_first_row_as_header=self.xlsx_first_row_as_header,
|
|
|
|
|
)
|
2025-12-01 09:43:26 +00:00
|
|
|
# Remove rows where all values are NaN (completely empty rows)
|
2025-11-19 09:27:17 +00:00
|
|
|
df.dropna(how="all", inplace=True)
|
|
|
|
|
|
feat(chat): open cited documents at the cited passage (#3700)
* feat(chat): open cited documents at the cited passage
Clicking a citation used to show only the parsed chunk. It now opens the
original file in the references panel, jumps to the cited place and
highlights it, narrowed to the sentence the citation supports.
Parsing records where each chunk sits in the original file
(chunks.source_locators): PDF page and paragraph boxes from the docreader,
MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and
EPUB chapters aligned in Go against any parser engine's Markdown; text
offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites
are followed with a line-diff remapper. Scanned PDF pages get their OCR
text as quotes at search time so a sentence can pick its page.
The frontend renders PDFs with pdf.js and every other previewable type
with the existing preview, then highlights with the CSS Custom Highlight
API in a highlighter yellow that stays visible against the brand green.
* fix(docreader): serialize source blocks outside main so tests import it
test_docx_tables stubs docreader.parser in sys.modules, so importing
docreader.main from a later test failed in the full unittest discover run.
The wire conversion now lives in docreader.source_wire, which needs only
the proto.
* fix(chat): tighten source locators from review
- Remap rewritten-line-for-line stretches by line so scanned PDF pages
no longer drift onto the wrong page.
- Keep the source preview mounted for citations in the file already open.
- Wait for in-flight PDF page renders before looking for the text layer.
- Count image rune offsets in one pass; stream xlsx rows up to maxUnits.
- Start text locators after a UTF-8 BOM, as viewers decode without it.
- Only re-query image children for scanned-page results.
- Seek audio citations without autoplay and drop stale seek listeners.
- Fix the SQLite migration's cross-reference to versioned/000113.
* fix(migrations): renumber chunk source locators after main's chunk_images
main took versioned 000113 and sqlite 000032 for chunk_images, so the
source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
|
|
|
# Process each row in the DataFrame. The index is the 0-based
|
|
|
|
|
# sheet row (rows are read from row 1 and dropped rows keep
|
|
|
|
|
# their labels), so index + 1 is the row number users see.
|
|
|
|
|
for row_index, row in df.iterrows():
|
2025-11-19 09:27:17 +00:00
|
|
|
page_content = []
|
2025-12-01 09:43:26 +00:00
|
|
|
# Build key-value pairs for non-null values
|
2025-11-19 09:27:17 +00:00
|
|
|
for k, v in row.items():
|
2026-06-24 03:40:46 +00:00
|
|
|
if pd.notna(v) and not _is_image_function(v):
|
2025-11-19 09:27:17 +00:00
|
|
|
page_content.append(f"{k}: {v}")
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Skip rows with no valid content
|
2025-11-19 09:27:17 +00:00
|
|
|
if not page_content:
|
|
|
|
|
continue
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Format row as comma-separated key-value pairs
|
2025-11-19 09:27:17 +00:00
|
|
|
content_row = ",".join(page_content) + "\n"
|
|
|
|
|
end += len(content_row)
|
|
|
|
|
text.append(content_row)
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Create a chunk for this row with position tracking
|
2025-11-19 09:27:17 +00:00
|
|
|
chunks.append(
|
|
|
|
|
Chunk(content=content_row, seq=len(chunks), start=start, end=end)
|
|
|
|
|
)
|
feat(chat): open cited documents at the cited passage (#3700)
* feat(chat): open cited documents at the cited passage
Clicking a citation used to show only the parsed chunk. It now opens the
original file in the references panel, jumps to the cited place and
highlights it, narrowed to the sentence the citation supports.
Parsing records where each chunk sits in the original file
(chunks.source_locators): PDF page and paragraph boxes from the docreader,
MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and
EPUB chapters aligned in Go against any parser engine's Markdown; text
offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites
are followed with a line-diff remapper. Scanned PDF pages get their OCR
text as quotes at search time so a sentence can pick its page.
The frontend renders PDFs with pdf.js and every other previewable type
with the existing preview, then highlights with the CSS Custom Highlight
API in a highlighter yellow that stays visible against the brand green.
* fix(docreader): serialize source blocks outside main so tests import it
test_docx_tables stubs docreader.parser in sys.modules, so importing
docreader.main from a later test failed in the full unittest discover run.
The wire conversion now lives in docreader.source_wire, which needs only
the proto.
* fix(chat): tighten source locators from review
- Remap rewritten-line-for-line stretches by line so scanned PDF pages
no longer drift onto the wrong page.
- Keep the source preview mounted for citations in the file already open.
- Wait for in-flight PDF page renders before looking for the text layer.
- Count image rune offsets in one pass; stream xlsx rows up to maxUnits.
- Start text locators after a UTF-8 BOM, as viewers decode without it.
- Only re-query image children for scanned-page results.
- Seek audio citations without autoplay and drop stale seek listeners.
- Fix the SQLite migration's cross-reference to versioned/000113.
* fix(migrations): renumber chunk source locators after main's chunk_images
main took versioned 000113 and sqlite 000032 for chunk_images, so the
source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
|
|
|
try:
|
|
|
|
|
row_number = int(row_index) + 1
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
row_number = 0
|
|
|
|
|
if row_number > 0:
|
|
|
|
|
source_blocks.append(
|
|
|
|
|
{
|
|
|
|
|
"start": start,
|
|
|
|
|
"end": end,
|
|
|
|
|
"locator": {
|
|
|
|
|
"type": "sheet",
|
|
|
|
|
"sheet": str(excel_sheet_name),
|
|
|
|
|
"row_start": row_number,
|
|
|
|
|
"row_end": row_number,
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
)
|
2025-11-19 09:27:17 +00:00
|
|
|
start = end
|
|
|
|
|
|
2025-12-01 09:43:26 +00:00
|
|
|
# Combine all text and return as Document
|
feat(chat): open cited documents at the cited passage (#3700)
* feat(chat): open cited documents at the cited passage
Clicking a citation used to show only the parsed chunk. It now opens the
original file in the references panel, jumps to the cited place and
highlights it, narrowed to the sentence the citation supports.
Parsing records where each chunk sits in the original file
(chunks.source_locators): PDF page and paragraph boxes from the docreader,
MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and
EPUB chapters aligned in Go against any parser engine's Markdown; text
offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites
are followed with a line-diff remapper. Scanned PDF pages get their OCR
text as quotes at search time so a sentence can pick its page.
The frontend renders PDFs with pdf.js and every other previewable type
with the existing preview, then highlights with the CSS Custom Highlight
API in a highlighter yellow that stays visible against the brand green.
* fix(docreader): serialize source blocks outside main so tests import it
test_docx_tables stubs docreader.parser in sys.modules, so importing
docreader.main from a later test failed in the full unittest discover run.
The wire conversion now lives in docreader.source_wire, which needs only
the proto.
* fix(chat): tighten source locators from review
- Remap rewritten-line-for-line stretches by line so scanned PDF pages
no longer drift onto the wrong page.
- Keep the source preview mounted for citations in the file already open.
- Wait for in-flight PDF page renders before looking for the text layer.
- Count image rune offsets in one pass; stream xlsx rows up to maxUnits.
- Start text locators after a UTF-8 BOM, as viewers decode without it.
- Only re-query image children for scanned-page results.
- Seek audio citations without autoplay and drop stale seek listeners.
- Fix the SQLite migration's cross-reference to versioned/000113.
* fix(migrations): renumber chunk source locators after main's chunk_images
main took versioned 000113 and sqlite 000032 for chunk_images, so the
source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
|
|
|
return Document(
|
|
|
|
|
content="".join(text), chunks=chunks, source_blocks=source_blocks
|
|
|
|
|
)
|
2025-11-19 09:27:17 +00:00
|
|
|
|
|
|
|
|
|
2026-07-25 20:00:41 +00:00
|
|
|
def _read_sheet_dataframe(
|
|
|
|
|
excel_file: pd.ExcelFile,
|
|
|
|
|
sheet_name: str,
|
|
|
|
|
xlsx_first_row_as_header: bool = False,
|
|
|
|
|
) -> pd.DataFrame:
|
2026-06-03 04:00:09 +00:00
|
|
|
"""Read a worksheet into a DataFrame with stable column labels."""
|
|
|
|
|
from openpyxl.utils import get_column_letter
|
|
|
|
|
|
2026-07-28 13:53:39 +00:00
|
|
|
# Keep row 1 as data by default for both XLSX and legacy XLS. Users can
|
|
|
|
|
# explicitly restore the historical behavior where row 1 supplies semantic
|
|
|
|
|
# labels for every row.
|
|
|
|
|
df = excel_file.parse(sheet_name=sheet_name, header=None)
|
|
|
|
|
if xlsx_first_row_as_header and len(df.index) >= 2:
|
|
|
|
|
df.columns = _stable_header_labels(df.iloc[0].tolist())
|
|
|
|
|
return df.iloc[1:].copy()
|
|
|
|
|
|
|
|
|
|
df.columns = [get_column_letter(idx + 1) for idx in range(len(df.columns))]
|
2026-06-03 04:00:09 +00:00
|
|
|
return df
|
|
|
|
|
|
|
|
|
|
|
2026-07-25 20:00:41 +00:00
|
|
|
def _stable_header_labels(values: List[object]) -> List[str]:
|
|
|
|
|
"""Build non-empty, unique labels from an explicitly selected header row."""
|
|
|
|
|
from openpyxl.utils import get_column_letter
|
|
|
|
|
|
|
|
|
|
labels: List[str] = []
|
|
|
|
|
counts: dict[str, int] = {}
|
|
|
|
|
for index, value in enumerate(values, start=1):
|
|
|
|
|
label = ""
|
|
|
|
|
if pd.notna(value) and not _is_image_function(value):
|
|
|
|
|
label = str(value).strip()
|
|
|
|
|
if not label:
|
|
|
|
|
label = get_column_letter(index)
|
|
|
|
|
|
|
|
|
|
count = counts.get(label, 0) + 1
|
|
|
|
|
counts[label] = count
|
|
|
|
|
labels.append(label if count == 1 else f"{label}_{count}")
|
|
|
|
|
return labels
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _parse_bool(value: Any) -> bool:
|
|
|
|
|
if isinstance(value, bool):
|
|
|
|
|
return value
|
|
|
|
|
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
def _prepare_xlsx_bytes(data: bytes) -> bytes:
|
|
|
|
|
repaired = repair_xlsx_bytes(data)
|
|
|
|
|
if repaired is not None:
|
|
|
|
|
data = repaired
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
readable = strip_unreadable_ranges_xlsx(data)
|
|
|
|
|
if readable is not None:
|
|
|
|
|
data = readable
|
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638)
* fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads
openpyxl raises TypeError('expected ... Fill') on <fill> elements with no
patternFill/gradientFill child — cloud-doc exporters, LabView and Acumatica
ERP emit these, upstream treats the workbooks as non-conformant, and both
parse engines died on files Excel opens fine (#3637).
Extend the zip-level repair module with sanitize_xlsx_styles: scan
xl/styles.xml, replace each malformed fill in place with a no-op
patternFill (index-preserving, so cellXf fillId references stay valid),
rewrite the zip only when something needed fixing. The builtin chain
retries the merge-cell prefill once on sanitized bytes; the markitdown
chain sanitizes xlsx/xlsm before MarkItDown sees them. Fixes #3637.
* fix(docreader): log sanitized fill count and never raise from the sanitizer
Fold review feedback into the styles sanitizer: a structured log line on
every repair (count included, doubling as the recurrence meter for #3637,
mirroring the sharedStrings repair log) and a never-raise wrapper so an
internal sanitizer failure passes the original bytes through instead of
trading the caller's error for its own.
* fix(docreader): read the workbook stylesheet by its exact part name
The styles sanitizer picked its part with a suffix match over a set of
zip member names, so a workbook that also carries
xl/richData/richStyles.xml (Excel 365 in-cell images / data types) got
the wrong part on some PYTHONHASHSEED values and skipped the repair.
Read xl/styles.xml, the one stylesheet openpyxl loads.
---------
Co-authored-by: vicfei <vicfei@users.noreply.github.com>
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:23:57 +00:00
|
|
|
try:
|
|
|
|
|
return fill_merged_cells_xlsx(data)
|
|
|
|
|
except TypeError:
|
|
|
|
|
# A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
|
|
|
|
|
# load_workbook raise before pandas ever sees the file (#3637).
|
|
|
|
|
# sanitize_xlsx_styles returns None when the fills are clean, in which
|
|
|
|
|
# case the TypeError has a different cause and must propagate.
|
|
|
|
|
sanitized = sanitize_xlsx_styles(data)
|
|
|
|
|
if sanitized is None:
|
|
|
|
|
raise
|
|
|
|
|
return fill_merged_cells_xlsx(sanitized)
|
2026-06-03 04:00:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
|
|
|
|
|
"""Open an Excel workbook with explicit engine selection and fallbacks."""
|
|
|
|
|
data = content
|
|
|
|
|
converted_via_soffice = False
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
|
ext = detect_excel_format(data)
|
|
|
|
|
if ext is None:
|
|
|
|
|
if converted_via_soffice:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"Excel file format cannot be determined, you must specify an engine manually."
|
|
|
|
|
)
|
|
|
|
|
try:
|
|
|
|
|
data = normalize_excel_bytes(data, file_type=file_type)
|
|
|
|
|
except ValueError as exc:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"Excel file format cannot be determined, you must specify an engine manually."
|
|
|
|
|
) from exc
|
|
|
|
|
converted_via_soffice = True
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if ext == "ods":
|
|
|
|
|
converted = convert_excel_to_xlsx_bytes(data, suffix=".ods")
|
|
|
|
|
if converted:
|
|
|
|
|
data = converted
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
engine = engine_for_format(ext)
|
|
|
|
|
if ext == "xlsx":
|
|
|
|
|
data = _prepare_xlsx_bytes(data)
|
|
|
|
|
engine = "openpyxl"
|
|
|
|
|
try:
|
|
|
|
|
return pd.ExcelFile(BytesIO(data), engine=engine)
|
|
|
|
|
except ImportError as exc:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"Excel engine {engine!r} is not available for .{ext} files"
|
|
|
|
|
) from exc
|
|
|
|
|
except KeyError as exc:
|
|
|
|
|
if "sharedStrings.xml" not in str(exc) or engine != "openpyxl":
|
|
|
|
|
raise
|
|
|
|
|
repaired = repair_xlsx_bytes(data)
|
|
|
|
|
if repaired is None:
|
|
|
|
|
raise
|
|
|
|
|
logger.info("Repaired XLSX sharedStrings packaging before parse")
|
|
|
|
|
data = _prepare_xlsx_bytes(repaired)
|
|
|
|
|
continue
|
|
|
|
|
except ValueError as exc:
|
|
|
|
|
if converted_via_soffice or "cannot be determined" not in str(exc):
|
|
|
|
|
raise
|
|
|
|
|
try:
|
|
|
|
|
data = normalize_excel_bytes(content, file_type=file_type)
|
|
|
|
|
except ValueError:
|
|
|
|
|
raise
|
|
|
|
|
converted_via_soffice = True
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
2025-11-19 09:27:17 +00:00
|
|
|
if __name__ == "__main__":
|
2025-12-01 09:43:26 +00:00
|
|
|
# Example usage: Parse an Excel file and display results
|
2025-11-19 09:27:17 +00:00
|
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
|
|
2025-12-01 09:43:26 +00:00
|
|
|
# Specify the path to your Excel file
|
2025-11-19 09:27:17 +00:00
|
|
|
your_file = "/path/to/your/file.xlsx"
|
|
|
|
|
parser = ExcelParser()
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Read and parse the Excel file
|
2025-11-19 09:27:17 +00:00
|
|
|
with open(your_file, "rb") as f:
|
|
|
|
|
content = f.read()
|
|
|
|
|
document = parser.parse_into_text(content)
|
2025-12-01 09:43:26 +00:00
|
|
|
|
|
|
|
|
# Display the full document content
|
2025-11-19 09:27:17 +00:00
|
|
|
logger.error(document.content)
|
|
|
|
|
|
2025-12-01 09:43:26 +00:00
|
|
|
# Display the first chunk as an example
|
2025-11-19 09:27:17 +00:00
|
|
|
for chunk in document.chunks:
|
|
|
|
|
logger.error(chunk.content)
|
2025-12-01 09:43:26 +00:00
|
|
|
break # Only show the first chunk
|