2026-06-03 04:00:09 +00:00
|
|
|
import io
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
import tempfile
|
|
|
|
|
import unittest
|
|
|
|
|
import zipfile
|
2026-08-12 04:23:46 +00:00
|
|
|
from unittest.mock import patch
|
2026-06-03 04:00:09 +00:00
|
|
|
|
|
|
|
|
import openpyxl
|
|
|
|
|
import pandas as pd
|
2026-08-12 04:23:46 +00:00
|
|
|
from openpyxl.chart import BarChart, Reference
|
2026-06-03 04:00:09 +00:00
|
|
|
|
|
|
|
|
from docreader.parser.excel_convert import detect_excel_format, engine_for_format
|
|
|
|
|
from docreader.parser.excel_parser import ExcelParser
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
from docreader.parser.markitdown_parser import StdMarkitdownParser
|
2026-06-03 04:00:09 +00:00
|
|
|
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
from docreader.parser.xlsx_repair import repair_xlsx_bytes, strip_unreadable_ranges_xlsx
|
2026-06-03 04:00:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _xlsx_with_phantom_shared_strings() -> bytes:
|
|
|
|
|
"""Workbook with inline strings but a dangling sharedStrings manifest entry."""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "hello"
|
|
|
|
|
ws["B1"] = 42
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
|
|
|
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
|
|
|
|
|
zin.extractall(tmpdir)
|
|
|
|
|
|
|
|
|
|
ct_path = f"{tmpdir}/[Content_Types].xml"
|
|
|
|
|
with open(ct_path, encoding="utf-8") as f:
|
|
|
|
|
ct = f.read()
|
|
|
|
|
override = (
|
|
|
|
|
'<Override PartName="/xl/sharedStrings.xml" '
|
|
|
|
|
'ContentType="application/vnd.openxmlformats-officedocument.'
|
|
|
|
|
'spreadsheetml.sharedStrings+xml"/>'
|
|
|
|
|
)
|
|
|
|
|
with open(ct_path, "w", encoding="utf-8") as f:
|
|
|
|
|
f.write(ct.replace("</Types>", override + "</Types>"))
|
|
|
|
|
|
|
|
|
|
out = io.BytesIO()
|
|
|
|
|
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
|
|
|
|
|
for root, _, files in os.walk(tmpdir):
|
|
|
|
|
for name in files:
|
|
|
|
|
path = os.path.join(root, name)
|
|
|
|
|
arc = os.path.relpath(path, tmpdir)
|
|
|
|
|
zout.write(path, arc)
|
|
|
|
|
return out.getvalue()
|
|
|
|
|
|
|
|
|
|
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
|
|
|
|
|
def _xlsx_with_sheet_fragment(fragment: str) -> bytes:
|
|
|
|
|
"""A two-row workbook with ``fragment`` spliced into sheet1 before pageMargins,
|
|
|
|
|
which is where OOXML puts data validations and scenarios."""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"], ws["B1"] = "name", "qty"
|
|
|
|
|
ws["A2"], ws["B2"] = "apple", 3
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
|
|
|
|
|
out = io.BytesIO()
|
|
|
|
|
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin, zipfile.ZipFile(
|
|
|
|
|
out, "w", zipfile.ZIP_DEFLATED
|
|
|
|
|
) as zout:
|
|
|
|
|
for info in zin.infolist():
|
|
|
|
|
data = zin.read(info.filename)
|
|
|
|
|
if info.filename == "xl/worksheets/sheet1.xml":
|
|
|
|
|
sheet = data.decode("utf-8")
|
|
|
|
|
at = sheet.index("<pageMargins")
|
|
|
|
|
data = (sheet[:at] + fragment + sheet[at:]).encode("utf-8")
|
|
|
|
|
zout.writestr(info, data)
|
|
|
|
|
return out.getvalue()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validation(sqref: str, formula: str = '"a,b"') -> str:
|
|
|
|
|
return (
|
|
|
|
|
f'<dataValidation type="list" allowBlank="1" sqref="{sqref}">'
|
|
|
|
|
f"<formula1>{formula}</formula1></dataValidation>"
|
|
|
|
|
)
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
class ExcelFormatDetectionTest(unittest.TestCase):
|
|
|
|
|
def test_detect_xlsx_and_engine(self):
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
content = bio.getvalue()
|
|
|
|
|
self.assertEqual(detect_excel_format(content), "xlsx")
|
|
|
|
|
self.assertEqual(engine_for_format("xlsx"), "openpyxl")
|
|
|
|
|
|
|
|
|
|
def test_detect_xls_magic(self):
|
|
|
|
|
content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
|
|
|
|
|
self.assertEqual(detect_excel_format(content), "xls")
|
|
|
|
|
self.assertEqual(engine_for_format("xls"), "xlrd")
|
|
|
|
|
|
|
|
|
|
def test_open_legacy_xls_bytes_with_xlsx_extension(self):
|
|
|
|
|
if not shutil.which("soffice"):
|
|
|
|
|
self.skipTest("LibreOffice not available")
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "legacy"
|
|
|
|
|
xlsx_bio = io.BytesIO()
|
|
|
|
|
wb.save(xlsx_bio)
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
|
|
|
src = os.path.join(tmpdir, "sheet.xlsx")
|
|
|
|
|
with open(src, "wb") as handle:
|
|
|
|
|
handle.write(xlsx_bio.getvalue())
|
|
|
|
|
subprocess.run(
|
|
|
|
|
[
|
|
|
|
|
"soffice",
|
|
|
|
|
"--headless",
|
|
|
|
|
"--convert-to",
|
|
|
|
|
"xls",
|
|
|
|
|
"--outdir",
|
|
|
|
|
tmpdir,
|
|
|
|
|
src,
|
|
|
|
|
],
|
|
|
|
|
check=True,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
)
|
|
|
|
|
xls_path = os.path.join(tmpdir, "sheet.xls")
|
|
|
|
|
with open(xls_path, "rb") as handle:
|
|
|
|
|
xls_bytes = handle.read()
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(file_name="fake.xlsx", file_type="xlsx").parse_into_text(
|
|
|
|
|
xls_bytes
|
|
|
|
|
)
|
|
|
|
|
self.assertIn("legacy", document.content)
|
|
|
|
|
|
|
|
|
|
|
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608)
* fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599)
A Feishu sheet/bitable exported as .xlsx could fail to ingest with
"TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>".
openpyxl reads a data validation's `sqref`, and a scenario list's, through a
converter that turns the real range-parse ValueError into that message and
aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx)
and in pandas alike.
Measured on openpyxl 3.1.5, the version docreader locks: a whole column
(`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an
out-of-bounds range each raise it. Conditional formatting goes through the
same converter, but openpyxl already catches it there and drops the rule
with a warning, so it is left alone.
strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the
same contract: it returns rewritten bytes only when some range would fail
openpyxl's own MultiCellRange constructor — the exact step that raises — and
removes just those elements, logging how many. Every other workbook passes
through byte for byte. Neither element changes a cell's value, so dropping
one costs text extraction nothing, while keeping it cost the document.
* fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too
MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column
data validation still failed that engine with FileConversionException
after the builtin engine was fixed (#3599). Run the same
strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown.
---------
Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
|
|
|
|
|
|
|
|
class XlsxUnreadableRangesTest(unittest.TestCase):
|
|
|
|
|
"""#3599: a range openpyxl cannot parse must not cost the whole document."""
|
|
|
|
|
|
|
|
|
|
def test_whole_column_validation_aborts_openpyxl_without_the_fix(self):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
|
|
|
|
|
)
|
|
|
|
|
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
|
|
|
|
|
openpyxl.load_workbook(io.BytesIO(broken), data_only=True)
|
|
|
|
|
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
|
|
|
|
|
pd.read_excel(io.BytesIO(broken), engine="openpyxl")
|
|
|
|
|
|
|
|
|
|
def test_drops_each_range_spelling_openpyxl_rejects(self):
|
|
|
|
|
# Measured on openpyxl 3.1.5; every one of these raised the reported
|
|
|
|
|
# TypeError before the fix.
|
|
|
|
|
for sqref in ("C:C", "1:1", "A1:B2,C3", "#REF!", "A1:XFD1048577"):
|
|
|
|
|
with self.subTest(sqref=sqref):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
f'<dataValidations count="1">{_validation(sqref)}</dataValidations>'
|
|
|
|
|
)
|
|
|
|
|
fixed = strip_unreadable_ranges_xlsx(broken)
|
|
|
|
|
self.assertIsNotNone(fixed)
|
|
|
|
|
df = pd.read_excel(io.BytesIO(fixed), header=None, engine="openpyxl")
|
|
|
|
|
self.assertEqual(df.values.tolist(), [["name", "qty"], ["apple", 3]])
|
|
|
|
|
|
|
|
|
|
def test_drops_a_scenario_list_with_an_unreadable_range(self):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
# The range belongs to the list, not to each scenario.
|
|
|
|
|
'<scenarios sqref="A:A"><scenario name="s" count="1">'
|
|
|
|
|
'<inputCells r="A1" val="1"/></scenario></scenarios>'
|
|
|
|
|
)
|
|
|
|
|
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
|
|
|
|
|
openpyxl.load_workbook(io.BytesIO(broken), data_only=True)
|
|
|
|
|
fixed = strip_unreadable_ranges_xlsx(broken)
|
|
|
|
|
self.assertIsNotNone(fixed)
|
|
|
|
|
openpyxl.load_workbook(io.BytesIO(fixed), data_only=True)
|
|
|
|
|
|
|
|
|
|
def test_keeps_a_readable_validation_next_to_an_unreadable_one(self):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
'<dataValidations count="2">'
|
|
|
|
|
f'{_validation("C:C")}{_validation("D2:D10", chr(34) + "keep" + chr(34))}'
|
|
|
|
|
"</dataValidations>"
|
|
|
|
|
)
|
|
|
|
|
fixed = strip_unreadable_ranges_xlsx(broken)
|
|
|
|
|
self.assertIsNotNone(fixed)
|
|
|
|
|
ws = openpyxl.load_workbook(io.BytesIO(fixed)).active
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
[str(dv.sqref) for dv in ws.data_validations.dataValidation], ["D2:D10"]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def test_leaves_a_readable_workbook_untouched(self):
|
|
|
|
|
fine = _xlsx_with_sheet_fragment(
|
|
|
|
|
f'<dataValidations count="1">{_validation("C2:C10")}</dataValidations>'
|
|
|
|
|
)
|
|
|
|
|
self.assertIsNone(strip_unreadable_ranges_xlsx(fine))
|
|
|
|
|
self.assertIsNone(strip_unreadable_ranges_xlsx(b"not a zip"))
|
|
|
|
|
|
|
|
|
|
def test_excel_parser_reads_a_workbook_with_a_whole_column_validation(self):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
|
|
|
|
|
)
|
|
|
|
|
document = ExcelParser(file_name="feishu.xlsx", file_type="xlsx").parse_into_text(
|
|
|
|
|
broken
|
|
|
|
|
)
|
|
|
|
|
self.assertIn("apple", document.content)
|
|
|
|
|
|
|
|
|
|
def test_markitdown_parser_reads_a_workbook_with_a_whole_column_validation(self):
|
|
|
|
|
broken = _xlsx_with_sheet_fragment(
|
|
|
|
|
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
|
|
|
|
|
)
|
|
|
|
|
document = StdMarkitdownParser(
|
|
|
|
|
file_name="feishu.xlsx", file_type="xlsx"
|
|
|
|
|
).parse_into_text(broken)
|
|
|
|
|
self.assertIn("apple", document.content)
|
|
|
|
|
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
class XlsxRepairTest(unittest.TestCase):
|
|
|
|
|
def test_repair_removes_phantom_shared_strings_reference(self):
|
|
|
|
|
broken = _xlsx_with_phantom_shared_strings()
|
|
|
|
|
with self.assertRaises(KeyError):
|
|
|
|
|
pd.read_excel(io.BytesIO(broken))
|
|
|
|
|
|
|
|
|
|
repaired = repair_xlsx_bytes(broken)
|
|
|
|
|
self.assertIsNotNone(repaired)
|
|
|
|
|
df = pd.read_excel(io.BytesIO(repaired), header=None)
|
|
|
|
|
self.assertEqual(df.values.tolist(), [["hello", 42]])
|
|
|
|
|
|
|
|
|
|
def test_repair_skips_when_shared_string_cells_need_table(self):
|
|
|
|
|
import xlsxwriter
|
|
|
|
|
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb = xlsxwriter.Workbook(bio, {"in_memory": True})
|
|
|
|
|
ws = wb.add_worksheet()
|
|
|
|
|
ws.write(0, 0, "hello")
|
|
|
|
|
wb.close()
|
|
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
|
|
|
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
|
|
|
|
|
zin.extractall(tmpdir)
|
|
|
|
|
os.remove(f"{tmpdir}/xl/sharedStrings.xml")
|
|
|
|
|
|
|
|
|
|
out = io.BytesIO()
|
|
|
|
|
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
|
|
|
|
|
for root, _, files in os.walk(tmpdir):
|
|
|
|
|
for name in files:
|
|
|
|
|
path = os.path.join(root, name)
|
|
|
|
|
arc = os.path.relpath(path, tmpdir)
|
|
|
|
|
zout.write(path, arc)
|
|
|
|
|
broken = out.getvalue()
|
|
|
|
|
|
|
|
|
|
self.assertIsNone(repair_xlsx_bytes(broken))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class XlsxMergeFillTest(unittest.TestCase):
|
2026-08-12 04:23:46 +00:00
|
|
|
@staticmethod
|
|
|
|
|
def _workbook_with_merged_cells_and_chart() -> bytes:
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "title"
|
|
|
|
|
ws.merge_cells("A1:B1")
|
|
|
|
|
ws.append(["category", "value"])
|
|
|
|
|
ws.append(["one", 1])
|
|
|
|
|
ws.append(["two", 2])
|
|
|
|
|
|
|
|
|
|
chart = BarChart()
|
|
|
|
|
chart.add_data(
|
|
|
|
|
Reference(ws, min_col=2, min_row=2, max_row=4),
|
|
|
|
|
titles_from_data=True,
|
|
|
|
|
)
|
|
|
|
|
chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
|
|
|
|
|
ws.add_chart(chart, "D2")
|
|
|
|
|
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
return bio.getvalue()
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
def test_fill_merged_cells_propagates_master_value(self):
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "title"
|
|
|
|
|
ws.merge_cells("A1:B1")
|
|
|
|
|
ws["A2"] = "left"
|
|
|
|
|
ws["B2"] = "right"
|
|
|
|
|
ws.merge_cells("A2:A3")
|
|
|
|
|
ws["B3"] = "only-b"
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
|
|
|
|
|
filled = fill_merged_cells_xlsx(bio.getvalue())
|
|
|
|
|
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
|
|
|
|
|
out_ws = out_wb.active
|
|
|
|
|
self.assertEqual(out_ws["B1"].value, "title")
|
|
|
|
|
self.assertEqual(out_ws["A3"].value, "left")
|
|
|
|
|
self.assertEqual(out_ws["B3"].value, "only-b")
|
|
|
|
|
|
2026-08-12 04:23:46 +00:00
|
|
|
def test_fill_merged_cells_does_not_serialize_charts(self):
|
|
|
|
|
content = self._workbook_with_merged_cells_and_chart()
|
|
|
|
|
|
|
|
|
|
with patch(
|
|
|
|
|
"openpyxl.chart._chart.ChartBase._write",
|
|
|
|
|
side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
|
|
|
|
|
):
|
|
|
|
|
filled = fill_merged_cells_xlsx(content)
|
|
|
|
|
|
|
|
|
|
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
|
|
|
|
|
out_ws = out_wb.active
|
|
|
|
|
self.assertEqual(out_ws["B1"].value, "title")
|
|
|
|
|
self.assertEqual(out_ws["A3"].value, "one")
|
|
|
|
|
self.assertEqual(out_ws["B4"].value, 2)
|
|
|
|
|
self.assertEqual(out_ws._charts, [])
|
|
|
|
|
|
|
|
|
|
def test_excel_parser_reads_merged_workbook_with_chart(self):
|
|
|
|
|
document = ExcelParser().parse_into_text(
|
|
|
|
|
self._workbook_with_merged_cells_and_chart()
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
self.assertIn("A: title,B: title", document.content)
|
|
|
|
|
self.assertIn("A: one,B: 1", document.content)
|
|
|
|
|
self.assertIn("A: two,B: 2", document.content)
|
|
|
|
|
|
|
|
|
|
def test_chart_workbook_without_merges_passes_through_unchanged(self):
|
|
|
|
|
content = self._workbook_with_merged_cells_and_chart()
|
|
|
|
|
wb = openpyxl.load_workbook(io.BytesIO(content))
|
|
|
|
|
wb.active.unmerge_cells("A1:B1")
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
content = bio.getvalue()
|
|
|
|
|
|
|
|
|
|
self.assertEqual(fill_merged_cells_xlsx(content), content)
|
|
|
|
|
|
|
|
|
|
def test_fill_does_not_serialize_dedicated_chart_sheets(self):
|
|
|
|
|
content = self._workbook_with_merged_cells_and_chart()
|
|
|
|
|
wb = openpyxl.load_workbook(io.BytesIO(content))
|
|
|
|
|
ws = wb.active
|
|
|
|
|
chart = BarChart()
|
|
|
|
|
chart.add_data(
|
|
|
|
|
Reference(ws, min_col=2, min_row=2, max_row=4),
|
|
|
|
|
titles_from_data=True,
|
|
|
|
|
)
|
|
|
|
|
chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
|
|
|
|
|
chart_sheet = wb.create_chartsheet("Summary")
|
|
|
|
|
chart_sheet.add_chart(chart)
|
|
|
|
|
wb.active = chart_sheet
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
|
|
|
|
|
with patch(
|
|
|
|
|
"openpyxl.chart._chart.ChartBase._write",
|
|
|
|
|
side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
|
|
|
|
|
):
|
|
|
|
|
filled = fill_merged_cells_xlsx(bio.getvalue())
|
|
|
|
|
|
|
|
|
|
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
|
|
|
|
|
self.assertEqual(out_wb.active._charts, [])
|
|
|
|
|
self.assertEqual(out_wb.active.title, "Sheet")
|
|
|
|
|
self.assertEqual(out_wb.chartsheets, [])
|
|
|
|
|
|
|
|
|
|
def test_fill_keeps_active_worksheet_after_removing_earlier_chart_sheet(self):
|
|
|
|
|
content = self._workbook_with_merged_cells_and_chart()
|
|
|
|
|
wb = openpyxl.load_workbook(io.BytesIO(content))
|
|
|
|
|
data_sheet = wb.active
|
|
|
|
|
chart = BarChart()
|
|
|
|
|
chart.add_data(
|
|
|
|
|
Reference(data_sheet, min_col=2, min_row=2, max_row=4),
|
|
|
|
|
titles_from_data=True,
|
|
|
|
|
)
|
|
|
|
|
chart_sheet = wb.create_chartsheet("Summary", 0)
|
|
|
|
|
chart_sheet.add_chart(chart)
|
|
|
|
|
wb.active = data_sheet
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
|
|
|
|
|
filled = fill_merged_cells_xlsx(bio.getvalue())
|
|
|
|
|
|
|
|
|
|
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
|
|
|
|
|
self.assertEqual(out_wb.active.title, "Sheet")
|
|
|
|
|
self.assertEqual(out_wb.chartsheets, [])
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
def test_parse_en_mergecell_workbook(self):
|
|
|
|
|
path = os.path.join(
|
|
|
|
|
os.path.dirname(__file__),
|
|
|
|
|
"..",
|
|
|
|
|
"..",
|
|
|
|
|
"testdata",
|
|
|
|
|
"rag_test",
|
|
|
|
|
"xlsx",
|
|
|
|
|
"en_mergecell.xlsx",
|
|
|
|
|
)
|
|
|
|
|
if not os.path.isfile(path):
|
|
|
|
|
self.skipTest("en_mergecell.xlsx fixture not available")
|
|
|
|
|
with open(path, "rb") as handle:
|
|
|
|
|
document = ExcelParser().parse_into_text(handle.read())
|
|
|
|
|
|
|
|
|
|
chunks = [chunk.content.strip() for chunk in document.chunks]
|
|
|
|
|
self.assertEqual(len(chunks), 12)
|
|
|
|
|
self.assertIn("A: A1", chunks[0])
|
|
|
|
|
self.assertIn("A: A2", chunks[1])
|
|
|
|
|
self.assertIn("B: B3", chunks[2])
|
|
|
|
|
self.assertNotIn("Unnamed:", document.content)
|
|
|
|
|
self.assertIn("A: A7", chunks[6])
|
|
|
|
|
self.assertIn("A: A7", chunks[7])
|
|
|
|
|
self.assertIn("D: D10", chunks[9])
|
|
|
|
|
|
|
|
|
|
|
2026-06-24 03:40:46 +00:00
|
|
|
class ExcelImageFilterTest(unittest.TestCase):
|
|
|
|
|
"""Tests for filtering embedded image function strings (#1779)."""
|
|
|
|
|
|
|
|
|
|
def _xlsx_with_image_functions(self) -> bytes:
|
|
|
|
|
"""Create an XLSX where image functions are stored as text values.
|
|
|
|
|
|
|
|
|
|
WPS embeds images using =DISPIMG("ID",1) which may appear as plain
|
|
|
|
|
text (not a formula) in some export scenarios.
|
|
|
|
|
"""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "Name"
|
|
|
|
|
ws["B1"] = "Photo"
|
|
|
|
|
ws["A2"] = "Alice"
|
|
|
|
|
ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
|
|
|
|
|
ws["A3"] = "Bob"
|
|
|
|
|
ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
|
|
|
|
|
ws["A4"] = "Charlie"
|
|
|
|
|
ws["B4"] = "real data"
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
return bio.getvalue()
|
|
|
|
|
|
|
|
|
|
def test_dispimg_text_values_are_excluded(self):
|
|
|
|
|
"""Image function strings stored as text must not appear in output."""
|
|
|
|
|
document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
|
|
|
|
|
self.assertNotIn("DISPIMG", document.content)
|
|
|
|
|
self.assertNotIn("_xlfn", document.content)
|
|
|
|
|
# Real data must still be present
|
|
|
|
|
self.assertIn("Alice", document.content)
|
|
|
|
|
self.assertIn("Bob", document.content)
|
|
|
|
|
self.assertIn("Charlie", document.content)
|
|
|
|
|
self.assertIn("real data", document.content)
|
|
|
|
|
|
|
|
|
|
def test_dispimg_with_equals_prefix(self):
|
|
|
|
|
"""=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "Name"
|
|
|
|
|
ws["B1"] = "Photo"
|
|
|
|
|
ws["A2"] = "Alice"
|
|
|
|
|
# Stored as text with = prefix (some exporters do this)
|
|
|
|
|
ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
# Note: openpyxl treats strings starting with = as formulas,
|
|
|
|
|
# so data_only=True in fill_merged_cells_xlsx will turn them to None.
|
|
|
|
|
# This test verifies the formula path also works correctly.
|
|
|
|
|
document = ExcelParser().parse_into_text(bio.getvalue())
|
|
|
|
|
self.assertNotIn("DISPIMG", document.content)
|
|
|
|
|
self.assertIn("Alice", document.content)
|
|
|
|
|
|
|
|
|
|
def test_image_function_variations(self):
|
|
|
|
|
"""Various image function patterns should all be filtered."""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
|
|
|
|
|
ws["A2"] = 'DISPIMG("ID_002",1)' # no prefix at all
|
|
|
|
|
ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
|
|
|
|
|
ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
|
|
|
|
|
ws["A5"] = "Normal text"
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
document = ExcelParser().parse_into_text(bio.getvalue())
|
|
|
|
|
self.assertNotIn("DISPIMG", document.content)
|
|
|
|
|
self.assertNotIn("IMAGE(", document.content)
|
|
|
|
|
self.assertIn("Normal text", document.content)
|
|
|
|
|
|
|
|
|
|
def test_real_world_wps_dispimg(self):
|
|
|
|
|
"""Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws["A1"] = "Product"
|
|
|
|
|
ws["B1"] = "Image"
|
|
|
|
|
ws["C1"] = "Price"
|
|
|
|
|
ws["A2"] = "Basic"
|
|
|
|
|
# This is the exact format shown in the issue screenshot
|
|
|
|
|
ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
|
|
|
|
|
ws["C2"] = "39.9"
|
|
|
|
|
ws["A3"] = "Pro"
|
|
|
|
|
ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
|
|
|
|
|
ws["C3"] = "99.9"
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
document = ExcelParser().parse_into_text(bio.getvalue())
|
|
|
|
|
self.assertNotIn("DISPIMG", document.content)
|
|
|
|
|
self.assertNotIn("ID_5A60F9ED", document.content)
|
|
|
|
|
self.assertIn("Basic", document.content)
|
|
|
|
|
self.assertIn("39.9", document.content)
|
|
|
|
|
self.assertIn("Pro", document.content)
|
|
|
|
|
self.assertIn("99.9", document.content)
|
|
|
|
|
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
class ExcelParserTest(unittest.TestCase):
|
2026-07-25 20:00:41 +00:00
|
|
|
@staticmethod
|
|
|
|
|
def _workbook_bytes(rows):
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
for row in rows:
|
|
|
|
|
ws.append(row)
|
|
|
|
|
bio = io.BytesIO()
|
|
|
|
|
wb.save(bio)
|
|
|
|
|
return bio.getvalue()
|
|
|
|
|
|
|
|
|
|
def test_xlsx_first_row_header_mode_repeats_column_context(self):
|
|
|
|
|
content = self._workbook_bytes(
|
|
|
|
|
[
|
|
|
|
|
["Name", "Age", "City"],
|
|
|
|
|
["Alice", 30, "Shenzhen"],
|
|
|
|
|
["Bob", 28, "Shanghai"],
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(
|
|
|
|
|
file_name="people.xlsx",
|
|
|
|
|
file_type="xlsx",
|
|
|
|
|
xlsx_first_row_as_header="true",
|
|
|
|
|
).parse_into_text(content)
|
|
|
|
|
|
|
|
|
|
chunks = [chunk.content.strip() for chunk in document.chunks]
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
chunks,
|
|
|
|
|
[
|
|
|
|
|
"Name: Alice,Age: 30,City: Shenzhen",
|
|
|
|
|
"Name: Bob,Age: 28,City: Shanghai",
|
|
|
|
|
],
|
|
|
|
|
)
|
|
|
|
|
self.assertNotIn("A: Name", document.content)
|
|
|
|
|
|
|
|
|
|
def test_xlsx_keeps_first_row_as_data_by_default(self):
|
|
|
|
|
content = self._workbook_bytes(
|
|
|
|
|
[
|
|
|
|
|
["Name", "City"],
|
|
|
|
|
["Alice", "Shenzhen"],
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(file_name="people.xlsx", file_type="xlsx").parse_into_text(
|
|
|
|
|
content
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
chunks = [chunk.content.strip() for chunk in document.chunks]
|
|
|
|
|
self.assertEqual(len(chunks), 2)
|
|
|
|
|
self.assertEqual(chunks[0], "A: Name,B: City")
|
|
|
|
|
self.assertEqual(chunks[1], "A: Alice,B: Shenzhen")
|
|
|
|
|
|
|
|
|
|
def test_single_row_xlsx_is_not_consumed_in_header_mode(self):
|
|
|
|
|
content = self._workbook_bytes([["Name", "Age", "City"]])
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(
|
|
|
|
|
file_name="single.xlsx",
|
|
|
|
|
file_type="xlsx",
|
|
|
|
|
xlsx_first_row_as_header=True,
|
|
|
|
|
).parse_into_text(content)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
[chunk.content.strip() for chunk in document.chunks],
|
|
|
|
|
["A: Name,B: Age,C: City"],
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def test_header_mode_generates_stable_labels_for_duplicate_and_empty_cells(self):
|
|
|
|
|
content = self._workbook_bytes(
|
|
|
|
|
[
|
|
|
|
|
["Name", "Name", None],
|
|
|
|
|
["Alice", "Alias", "Shenzhen"],
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(
|
|
|
|
|
file_name="people.xlsx",
|
|
|
|
|
file_type="xlsx",
|
|
|
|
|
xlsx_first_row_as_header="yes",
|
|
|
|
|
).parse_into_text(content)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
[chunk.content.strip() for chunk in document.chunks],
|
|
|
|
|
["Name: Alice,Name_2: Alias,C: Shenzhen"],
|
|
|
|
|
)
|
|
|
|
|
|
2026-07-28 13:53:39 +00:00
|
|
|
def test_xlsx_explicit_false_override_keeps_first_row_as_data(self):
|
|
|
|
|
content = self._workbook_bytes(
|
|
|
|
|
[
|
|
|
|
|
["Name", "Age"],
|
|
|
|
|
["Alice", 30],
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
document = ExcelParser(
|
|
|
|
|
file_name="people.xlsx",
|
|
|
|
|
file_type="xlsx",
|
|
|
|
|
xlsx_first_row_as_header="false",
|
|
|
|
|
).parse_into_text(content)
|
|
|
|
|
|
|
|
|
|
chunks = [chunk.content.strip() for chunk in document.chunks]
|
|
|
|
|
self.assertEqual(chunks[0], "A: Name,B: Age")
|
|
|
|
|
self.assertEqual(chunks[1], "A: Alice,B: 30")
|
|
|
|
|
|
2026-06-03 04:00:09 +00:00
|
|
|
def test_parse_phantom_shared_strings_workbook(self):
|
|
|
|
|
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
|
|
|
|
|
self.assertIn("hello", document.content)
|
|
|
|
|
self.assertIn("42", document.content)
|
|
|
|
|
self.assertGreater(len(document.chunks), 0)
|
|
|
|
|
|
|
|
|
|
def test_parse_en_calcchain_shared_strings_case(self):
|
|
|
|
|
path = os.path.join(
|
|
|
|
|
os.path.dirname(__file__),
|
|
|
|
|
"..",
|
|
|
|
|
"..",
|
|
|
|
|
"testdata",
|
|
|
|
|
"rag_test",
|
|
|
|
|
"xlsx",
|
|
|
|
|
"en_calcchain.xlsx",
|
|
|
|
|
)
|
|
|
|
|
if not os.path.isfile(path):
|
|
|
|
|
self.skipTest("en_calcchain.xlsx fixture not available")
|
|
|
|
|
with open(path, "rb") as f:
|
|
|
|
|
document = ExcelParser().parse_into_text(f.read())
|
|
|
|
|
self.assertGreater(len(document.content), 0)
|
|
|
|
|
self.assertGreater(len(document.chunks), 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
unittest.main()
|
feat(chat): open cited documents at the cited passage (#3700)
* feat(chat): open cited documents at the cited passage
Clicking a citation used to show only the parsed chunk. It now opens the
original file in the references panel, jumps to the cited place and
highlights it, narrowed to the sentence the citation supports.
Parsing records where each chunk sits in the original file
(chunks.source_locators): PDF page and paragraph boxes from the docreader,
MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and
EPUB chapters aligned in Go against any parser engine's Markdown; text
offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites
are followed with a line-diff remapper. Scanned PDF pages get their OCR
text as quotes at search time so a sentence can pick its page.
The frontend renders PDFs with pdf.js and every other previewable type
with the existing preview, then highlights with the CSS Custom Highlight
API in a highlighter yellow that stays visible against the brand green.
* fix(docreader): serialize source blocks outside main so tests import it
test_docx_tables stubs docreader.parser in sys.modules, so importing
docreader.main from a later test failed in the full unittest discover run.
The wire conversion now lives in docreader.source_wire, which needs only
the proto.
* fix(chat): tighten source locators from review
- Remap rewritten-line-for-line stretches by line so scanned PDF pages
no longer drift onto the wrong page.
- Keep the source preview mounted for citations in the file already open.
- Wait for in-flight PDF page renders before looking for the text layer.
- Count image rune offsets in one pass; stream xlsx rows up to maxUnits.
- Start text locators after a UTF-8 BOM, as viewers decode without it.
- Only re-query image children for scanned-page results.
- Seek audio citations without autoplay and drop stale seek listeners.
- Fix the SQLite migration's cross-reference to versioned/000113.
* fix(migrations): renumber chunk source locators after main's chunk_images
main took versioned 000113 and sqlite 000032 for chunk_images, so the
source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
class ExcelSourceBlocksTest(unittest.TestCase):
|
|
|
|
|
def test_rows_carry_sheet_and_row_number(self):
|
|
|
|
|
import io
|
|
|
|
|
|
|
|
|
|
import openpyxl
|
|
|
|
|
|
|
|
|
|
from docreader.parser.excel_parser import ExcelParser
|
|
|
|
|
|
|
|
|
|
wb = openpyxl.Workbook()
|
|
|
|
|
ws = wb.active
|
|
|
|
|
ws.title = "明细"
|
|
|
|
|
ws["B3"] = "螺栓"
|
|
|
|
|
ws["A5"] = "合计"
|
|
|
|
|
buf = io.BytesIO()
|
|
|
|
|
wb.save(buf)
|
|
|
|
|
|
|
|
|
|
doc = ExcelParser(file_name="a.xlsx", file_type="xlsx").parse_into_text(buf.getvalue())
|
|
|
|
|
rows = [(b["locator"]["sheet"], b["locator"]["row_start"]) for b in doc.source_blocks]
|
|
|
|
|
self.assertEqual(rows, [("明细", 3), ("明细", 5)])
|
|
|
|
|
first = doc.source_blocks[0]
|
|
|
|
|
self.assertEqual(doc.content[first["start"]:first["end"]], "B: 螺栓\n")
|