WeKnora/docreader/tests/test_excel_parser.py

646 lines
24 KiB
Python
Raw Normal View History

import io
import os
import shutil
import subprocess
import tempfile
import unittest
import zipfile
from unittest.mock import patch
import openpyxl
import pandas as pd
from openpyxl.chart import BarChart, Reference
from docreader.parser.excel_convert import detect_excel_format, engine_for_format
from docreader.parser.excel_parser import ExcelParser
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608) * fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) A Feishu sheet/bitable exported as .xlsx could fail to ingest with "TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>". openpyxl reads a data validation's `sqref`, and a scenario list's, through a converter that turns the real range-parse ValueError into that message and aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx) and in pandas alike. Measured on openpyxl 3.1.5, the version docreader locks: a whole column (`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an out-of-bounds range each raise it. Conditional formatting goes through the same converter, but openpyxl already catches it there and drops the rule with a warning, so it is left alone. strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the same contract: it returns rewritten bytes only when some range would fail openpyxl's own MultiCellRange constructor — the exact step that raises — and removes just those elements, logging how many. Every other workbook passes through byte for byte. Neither element changes a cell's value, so dropping one costs text extraction nothing, while keeping it cost the document. * fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column data validation still failed that engine with FileConversionException after the builtin engine was fixed (#3599). Run the same strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
from docreader.parser.markitdown_parser import StdMarkitdownParser
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608) * fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) A Feishu sheet/bitable exported as .xlsx could fail to ingest with "TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>". openpyxl reads a data validation's `sqref`, and a scenario list's, through a converter that turns the real range-parse ValueError into that message and aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx) and in pandas alike. Measured on openpyxl 3.1.5, the version docreader locks: a whole column (`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an out-of-bounds range each raise it. Conditional formatting goes through the same converter, but openpyxl already catches it there and drops the rule with a warning, so it is left alone. strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the same contract: it returns rewritten bytes only when some range would fail openpyxl's own MultiCellRange constructor — the exact step that raises — and removes just those elements, logging how many. Every other workbook passes through byte for byte. Neither element changes a cell's value, so dropping one costs text extraction nothing, while keeping it cost the document. * fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column data validation still failed that engine with FileConversionException after the builtin engine was fixed (#3599). Run the same strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
from docreader.parser.xlsx_repair import repair_xlsx_bytes, strip_unreadable_ranges_xlsx
def _xlsx_with_phantom_shared_strings() -> bytes:
"""Workbook with inline strings but a dangling sharedStrings manifest entry."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "hello"
ws["B1"] = 42
bio = io.BytesIO()
wb.save(bio)
with tempfile.TemporaryDirectory() as tmpdir:
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
zin.extractall(tmpdir)
ct_path = f"{tmpdir}/[Content_Types].xml"
with open(ct_path, encoding="utf-8") as f:
ct = f.read()
override = (
'<Override PartName="/xl/sharedStrings.xml" '
'ContentType="application/vnd.openxmlformats-officedocument.'
'spreadsheetml.sharedStrings+xml"/>'
)
with open(ct_path, "w", encoding="utf-8") as f:
f.write(ct.replace("</Types>", override + "</Types>"))
out = io.BytesIO()
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
for root, _, files in os.walk(tmpdir):
for name in files:
path = os.path.join(root, name)
arc = os.path.relpath(path, tmpdir)
zout.write(path, arc)
return out.getvalue()
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608) * fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) A Feishu sheet/bitable exported as .xlsx could fail to ingest with "TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>". openpyxl reads a data validation's `sqref`, and a scenario list's, through a converter that turns the real range-parse ValueError into that message and aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx) and in pandas alike. Measured on openpyxl 3.1.5, the version docreader locks: a whole column (`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an out-of-bounds range each raise it. Conditional formatting goes through the same converter, but openpyxl already catches it there and drops the rule with a warning, so it is left alone. strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the same contract: it returns rewritten bytes only when some range would fail openpyxl's own MultiCellRange constructor — the exact step that raises — and removes just those elements, logging how many. Every other workbook passes through byte for byte. Neither element changes a cell's value, so dropping one costs text extraction nothing, while keeping it cost the document. * fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column data validation still failed that engine with FileConversionException after the builtin engine was fixed (#3599). Run the same strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
def _xlsx_with_sheet_fragment(fragment: str) -> bytes:
"""A two-row workbook with ``fragment`` spliced into sheet1 before pageMargins,
which is where OOXML puts data validations and scenarios."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"], ws["B1"] = "name", "qty"
ws["A2"], ws["B2"] = "apple", 3
bio = io.BytesIO()
wb.save(bio)
out = io.BytesIO()
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin, zipfile.ZipFile(
out, "w", zipfile.ZIP_DEFLATED
) as zout:
for info in zin.infolist():
data = zin.read(info.filename)
if info.filename == "xl/worksheets/sheet1.xml":
sheet = data.decode("utf-8")
at = sheet.index("<pageMargins")
data = (sheet[:at] + fragment + sheet[at:]).encode("utf-8")
zout.writestr(info, data)
return out.getvalue()
def _validation(sqref: str, formula: str = '"a,b"') -> str:
return (
f'<dataValidation type="list" allowBlank="1" sqref="{sqref}">'
f"<formula1>{formula}</formula1></dataValidation>"
)
class ExcelFormatDetectionTest(unittest.TestCase):
def test_detect_xlsx_and_engine(self):
wb = openpyxl.Workbook()
bio = io.BytesIO()
wb.save(bio)
content = bio.getvalue()
self.assertEqual(detect_excel_format(content), "xlsx")
self.assertEqual(engine_for_format("xlsx"), "openpyxl")
def test_detect_xls_magic(self):
content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
self.assertEqual(detect_excel_format(content), "xls")
self.assertEqual(engine_for_format("xls"), "xlrd")
def test_open_legacy_xls_bytes_with_xlsx_extension(self):
if not shutil.which("soffice"):
self.skipTest("LibreOffice not available")
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "legacy"
xlsx_bio = io.BytesIO()
wb.save(xlsx_bio)
with tempfile.TemporaryDirectory() as tmpdir:
src = os.path.join(tmpdir, "sheet.xlsx")
with open(src, "wb") as handle:
handle.write(xlsx_bio.getvalue())
subprocess.run(
[
"soffice",
"--headless",
"--convert-to",
"xls",
"--outdir",
tmpdir,
src,
],
check=True,
capture_output=True,
)
xls_path = os.path.join(tmpdir, "sheet.xls")
with open(xls_path, "rb") as handle:
xls_bytes = handle.read()
document = ExcelParser(file_name="fake.xlsx", file_type="xlsx").parse_into_text(
xls_bytes
)
self.assertIn("legacy", document.content)
fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) (#3608) * fix(docreader): drop data validation ranges openpyxl cannot parse instead of failing the sheet (#3599) A Feishu sheet/bitable exported as .xlsx could fail to ingest with "TypeError: expected <class 'openpyxl.worksheet.cell_range.MultiCellRange'>". openpyxl reads a data validation's `sqref`, and a scenario list's, through a converter that turns the real range-parse ValueError into that message and aborts the whole workbook — in load_workbook (used by fill_merged_cells_xlsx) and in pandas alike. Measured on openpyxl 3.1.5, the version docreader locks: a whole column (`C:C`), a whole row (`1:1`), a comma-separated list, `#REF!` and an out-of-bounds range each raise it. Conditional formatting goes through the same converter, but openpyxl already catches it there and drops the rule with a warning, so it is left alone. strip_unreadable_ranges_xlsx sits next to repair_xlsx_bytes and follows the same contract: it returns rewritten bytes only when some range would fail openpyxl's own MultiCellRange constructor — the exact step that raises — and removes just those elements, logging how many. Every other workbook passes through byte for byte. Neither element changes a cell's value, so dropping one costs text extraction nothing, while keeping it cost the document. * fix(docreader): strip unreadable ranges on the MarkItDown xlsx path too MarkItDown reads spreadsheets through pandas/openpyxl, so a whole-column data validation still failed that engine with FileConversionException after the builtin engine was fixed (#3599). Run the same strip_unreadable_ranges_xlsx pre-step before handing .xlsx to MarkItDown. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>
2026-09-25 14:29:27 +00:00
class XlsxUnreadableRangesTest(unittest.TestCase):
"""#3599: a range openpyxl cannot parse must not cost the whole document."""
def test_whole_column_validation_aborts_openpyxl_without_the_fix(self):
broken = _xlsx_with_sheet_fragment(
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
)
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
openpyxl.load_workbook(io.BytesIO(broken), data_only=True)
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
pd.read_excel(io.BytesIO(broken), engine="openpyxl")
def test_drops_each_range_spelling_openpyxl_rejects(self):
# Measured on openpyxl 3.1.5; every one of these raised the reported
# TypeError before the fix.
for sqref in ("C:C", "1:1", "A1:B2,C3", "#REF!", "A1:XFD1048577"):
with self.subTest(sqref=sqref):
broken = _xlsx_with_sheet_fragment(
f'<dataValidations count="1">{_validation(sqref)}</dataValidations>'
)
fixed = strip_unreadable_ranges_xlsx(broken)
self.assertIsNotNone(fixed)
df = pd.read_excel(io.BytesIO(fixed), header=None, engine="openpyxl")
self.assertEqual(df.values.tolist(), [["name", "qty"], ["apple", 3]])
def test_drops_a_scenario_list_with_an_unreadable_range(self):
broken = _xlsx_with_sheet_fragment(
# The range belongs to the list, not to each scenario.
'<scenarios sqref="A:A"><scenario name="s" count="1">'
'<inputCells r="A1" val="1"/></scenario></scenarios>'
)
with self.assertRaisesRegex(TypeError, "MultiCellRange"):
openpyxl.load_workbook(io.BytesIO(broken), data_only=True)
fixed = strip_unreadable_ranges_xlsx(broken)
self.assertIsNotNone(fixed)
openpyxl.load_workbook(io.BytesIO(fixed), data_only=True)
def test_keeps_a_readable_validation_next_to_an_unreadable_one(self):
broken = _xlsx_with_sheet_fragment(
'<dataValidations count="2">'
f'{_validation("C:C")}{_validation("D2:D10", chr(34) + "keep" + chr(34))}'
"</dataValidations>"
)
fixed = strip_unreadable_ranges_xlsx(broken)
self.assertIsNotNone(fixed)
ws = openpyxl.load_workbook(io.BytesIO(fixed)).active
self.assertEqual(
[str(dv.sqref) for dv in ws.data_validations.dataValidation], ["D2:D10"]
)
def test_leaves_a_readable_workbook_untouched(self):
fine = _xlsx_with_sheet_fragment(
f'<dataValidations count="1">{_validation("C2:C10")}</dataValidations>'
)
self.assertIsNone(strip_unreadable_ranges_xlsx(fine))
self.assertIsNone(strip_unreadable_ranges_xlsx(b"not a zip"))
def test_excel_parser_reads_a_workbook_with_a_whole_column_validation(self):
broken = _xlsx_with_sheet_fragment(
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
)
document = ExcelParser(file_name="feishu.xlsx", file_type="xlsx").parse_into_text(
broken
)
self.assertIn("apple", document.content)
def test_markitdown_parser_reads_a_workbook_with_a_whole_column_validation(self):
broken = _xlsx_with_sheet_fragment(
f'<dataValidations count="1">{_validation("C:C")}</dataValidations>'
)
document = StdMarkitdownParser(
file_name="feishu.xlsx", file_type="xlsx"
).parse_into_text(broken)
self.assertIn("apple", document.content)
class XlsxRepairTest(unittest.TestCase):
def test_repair_removes_phantom_shared_strings_reference(self):
broken = _xlsx_with_phantom_shared_strings()
with self.assertRaises(KeyError):
pd.read_excel(io.BytesIO(broken))
repaired = repair_xlsx_bytes(broken)
self.assertIsNotNone(repaired)
df = pd.read_excel(io.BytesIO(repaired), header=None)
self.assertEqual(df.values.tolist(), [["hello", 42]])
def test_repair_skips_when_shared_string_cells_need_table(self):
import xlsxwriter
bio = io.BytesIO()
wb = xlsxwriter.Workbook(bio, {"in_memory": True})
ws = wb.add_worksheet()
ws.write(0, 0, "hello")
wb.close()
with tempfile.TemporaryDirectory() as tmpdir:
with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
zin.extractall(tmpdir)
os.remove(f"{tmpdir}/xl/sharedStrings.xml")
out = io.BytesIO()
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
for root, _, files in os.walk(tmpdir):
for name in files:
path = os.path.join(root, name)
arc = os.path.relpath(path, tmpdir)
zout.write(path, arc)
broken = out.getvalue()
self.assertIsNone(repair_xlsx_bytes(broken))
class XlsxMergeFillTest(unittest.TestCase):
@staticmethod
def _workbook_with_merged_cells_and_chart() -> bytes:
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "title"
ws.merge_cells("A1:B1")
ws.append(["category", "value"])
ws.append(["one", 1])
ws.append(["two", 2])
chart = BarChart()
chart.add_data(
Reference(ws, min_col=2, min_row=2, max_row=4),
titles_from_data=True,
)
chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
ws.add_chart(chart, "D2")
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def test_fill_merged_cells_propagates_master_value(self):
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "title"
ws.merge_cells("A1:B1")
ws["A2"] = "left"
ws["B2"] = "right"
ws.merge_cells("A2:A3")
ws["B3"] = "only-b"
bio = io.BytesIO()
wb.save(bio)
filled = fill_merged_cells_xlsx(bio.getvalue())
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
out_ws = out_wb.active
self.assertEqual(out_ws["B1"].value, "title")
self.assertEqual(out_ws["A3"].value, "left")
self.assertEqual(out_ws["B3"].value, "only-b")
def test_fill_merged_cells_does_not_serialize_charts(self):
content = self._workbook_with_merged_cells_and_chart()
with patch(
"openpyxl.chart._chart.ChartBase._write",
side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
):
filled = fill_merged_cells_xlsx(content)
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
out_ws = out_wb.active
self.assertEqual(out_ws["B1"].value, "title")
self.assertEqual(out_ws["A3"].value, "one")
self.assertEqual(out_ws["B4"].value, 2)
self.assertEqual(out_ws._charts, [])
def test_excel_parser_reads_merged_workbook_with_chart(self):
document = ExcelParser().parse_into_text(
self._workbook_with_merged_cells_and_chart()
)
self.assertIn("A: title,B: title", document.content)
self.assertIn("A: one,B: 1", document.content)
self.assertIn("A: two,B: 2", document.content)
def test_chart_workbook_without_merges_passes_through_unchanged(self):
content = self._workbook_with_merged_cells_and_chart()
wb = openpyxl.load_workbook(io.BytesIO(content))
wb.active.unmerge_cells("A1:B1")
bio = io.BytesIO()
wb.save(bio)
content = bio.getvalue()
self.assertEqual(fill_merged_cells_xlsx(content), content)
def test_fill_does_not_serialize_dedicated_chart_sheets(self):
content = self._workbook_with_merged_cells_and_chart()
wb = openpyxl.load_workbook(io.BytesIO(content))
ws = wb.active
chart = BarChart()
chart.add_data(
Reference(ws, min_col=2, min_row=2, max_row=4),
titles_from_data=True,
)
chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
chart_sheet = wb.create_chartsheet("Summary")
chart_sheet.add_chart(chart)
wb.active = chart_sheet
bio = io.BytesIO()
wb.save(bio)
with patch(
"openpyxl.chart._chart.ChartBase._write",
side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
):
filled = fill_merged_cells_xlsx(bio.getvalue())
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
self.assertEqual(out_wb.active._charts, [])
self.assertEqual(out_wb.active.title, "Sheet")
self.assertEqual(out_wb.chartsheets, [])
def test_fill_keeps_active_worksheet_after_removing_earlier_chart_sheet(self):
content = self._workbook_with_merged_cells_and_chart()
wb = openpyxl.load_workbook(io.BytesIO(content))
data_sheet = wb.active
chart = BarChart()
chart.add_data(
Reference(data_sheet, min_col=2, min_row=2, max_row=4),
titles_from_data=True,
)
chart_sheet = wb.create_chartsheet("Summary", 0)
chart_sheet.add_chart(chart)
wb.active = data_sheet
bio = io.BytesIO()
wb.save(bio)
filled = fill_merged_cells_xlsx(bio.getvalue())
out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
self.assertEqual(out_wb.active.title, "Sheet")
self.assertEqual(out_wb.chartsheets, [])
def test_parse_en_mergecell_workbook(self):
path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"xlsx",
"en_mergecell.xlsx",
)
if not os.path.isfile(path):
self.skipTest("en_mergecell.xlsx fixture not available")
with open(path, "rb") as handle:
document = ExcelParser().parse_into_text(handle.read())
chunks = [chunk.content.strip() for chunk in document.chunks]
self.assertEqual(len(chunks), 12)
self.assertIn("A: A1", chunks[0])
self.assertIn("A: A2", chunks[1])
self.assertIn("B: B3", chunks[2])
self.assertNotIn("Unnamed:", document.content)
self.assertIn("A: A7", chunks[6])
self.assertIn("A: A7", chunks[7])
self.assertIn("D: D10", chunks[9])
class ExcelImageFilterTest(unittest.TestCase):
"""Tests for filtering embedded image function strings (#1779)."""
def _xlsx_with_image_functions(self) -> bytes:
"""Create an XLSX where image functions are stored as text values.
WPS embeds images using =DISPIMG("ID",1) which may appear as plain
text (not a formula) in some export scenarios.
"""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
ws["A3"] = "Bob"
ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
ws["A4"] = "Charlie"
ws["B4"] = "real data"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def test_dispimg_text_values_are_excluded(self):
"""Image function strings stored as text must not appear in output."""
document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("_xlfn", document.content)
# Real data must still be present
self.assertIn("Alice", document.content)
self.assertIn("Bob", document.content)
self.assertIn("Charlie", document.content)
self.assertIn("real data", document.content)
def test_dispimg_with_equals_prefix(self):
"""=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Name"
ws["B1"] = "Photo"
ws["A2"] = "Alice"
# Stored as text with = prefix (some exporters do this)
ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
bio = io.BytesIO()
wb.save(bio)
# Note: openpyxl treats strings starting with = as formulas,
# so data_only=True in fill_merged_cells_xlsx will turn them to None.
# This test verifies the formula path also works correctly.
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertIn("Alice", document.content)
def test_image_function_variations(self):
"""Various image function patterns should all be filtered."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
ws["A2"] = 'DISPIMG("ID_002",1)' # no prefix at all
ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
ws["A5"] = "Normal text"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("IMAGE(", document.content)
self.assertIn("Normal text", document.content)
def test_real_world_wps_dispimg(self):
"""Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
wb = openpyxl.Workbook()
ws = wb.active
ws["A1"] = "Product"
ws["B1"] = "Image"
ws["C1"] = "Price"
ws["A2"] = "Basic"
# This is the exact format shown in the issue screenshot
ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
ws["C2"] = "39.9"
ws["A3"] = "Pro"
ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
ws["C3"] = "99.9"
bio = io.BytesIO()
wb.save(bio)
document = ExcelParser().parse_into_text(bio.getvalue())
self.assertNotIn("DISPIMG", document.content)
self.assertNotIn("ID_5A60F9ED", document.content)
self.assertIn("Basic", document.content)
self.assertIn("39.9", document.content)
self.assertIn("Pro", document.content)
self.assertIn("99.9", document.content)
class ExcelParserTest(unittest.TestCase):
@staticmethod
def _workbook_bytes(rows):
wb = openpyxl.Workbook()
ws = wb.active
for row in rows:
ws.append(row)
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def test_xlsx_first_row_header_mode_repeats_column_context(self):
content = self._workbook_bytes(
[
["Name", "Age", "City"],
["Alice", 30, "Shenzhen"],
["Bob", 28, "Shanghai"],
]
)
document = ExcelParser(
file_name="people.xlsx",
file_type="xlsx",
xlsx_first_row_as_header="true",
).parse_into_text(content)
chunks = [chunk.content.strip() for chunk in document.chunks]
self.assertEqual(
chunks,
[
"Name: Alice,Age: 30,City: Shenzhen",
"Name: Bob,Age: 28,City: Shanghai",
],
)
self.assertNotIn("A: Name", document.content)
def test_xlsx_keeps_first_row_as_data_by_default(self):
content = self._workbook_bytes(
[
["Name", "City"],
["Alice", "Shenzhen"],
]
)
document = ExcelParser(file_name="people.xlsx", file_type="xlsx").parse_into_text(
content
)
chunks = [chunk.content.strip() for chunk in document.chunks]
self.assertEqual(len(chunks), 2)
self.assertEqual(chunks[0], "A: Name,B: City")
self.assertEqual(chunks[1], "A: Alice,B: Shenzhen")
def test_single_row_xlsx_is_not_consumed_in_header_mode(self):
content = self._workbook_bytes([["Name", "Age", "City"]])
document = ExcelParser(
file_name="single.xlsx",
file_type="xlsx",
xlsx_first_row_as_header=True,
).parse_into_text(content)
self.assertEqual(
[chunk.content.strip() for chunk in document.chunks],
["A: Name,B: Age,C: City"],
)
def test_header_mode_generates_stable_labels_for_duplicate_and_empty_cells(self):
content = self._workbook_bytes(
[
["Name", "Name", None],
["Alice", "Alias", "Shenzhen"],
]
)
document = ExcelParser(
file_name="people.xlsx",
file_type="xlsx",
xlsx_first_row_as_header="yes",
).parse_into_text(content)
self.assertEqual(
[chunk.content.strip() for chunk in document.chunks],
["Name: Alice,Name_2: Alias,C: Shenzhen"],
)
def test_xlsx_explicit_false_override_keeps_first_row_as_data(self):
content = self._workbook_bytes(
[
["Name", "Age"],
["Alice", 30],
]
)
document = ExcelParser(
file_name="people.xlsx",
file_type="xlsx",
xlsx_first_row_as_header="false",
).parse_into_text(content)
chunks = [chunk.content.strip() for chunk in document.chunks]
self.assertEqual(chunks[0], "A: Name,B: Age")
self.assertEqual(chunks[1], "A: Alice,B: 30")
def test_parse_phantom_shared_strings_workbook(self):
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
self.assertIn("hello", document.content)
self.assertIn("42", document.content)
self.assertGreater(len(document.chunks), 0)
def test_parse_en_calcchain_shared_strings_case(self):
path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"testdata",
"rag_test",
"xlsx",
"en_calcchain.xlsx",
)
if not os.path.isfile(path):
self.skipTest("en_calcchain.xlsx fixture not available")
with open(path, "rb") as f:
document = ExcelParser().parse_into_text(f.read())
self.assertGreater(len(document.content), 0)
self.assertGreater(len(document.chunks), 0)
if __name__ == "__main__":
unittest.main()
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
class ExcelSourceBlocksTest(unittest.TestCase):
def test_rows_carry_sheet_and_row_number(self):
import io
import openpyxl
from docreader.parser.excel_parser import ExcelParser
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "明细"
ws["B3"] = "螺栓"
ws["A5"] = "合计"
buf = io.BytesIO()
wb.save(buf)
doc = ExcelParser(file_name="a.xlsx", file_type="xlsx").parse_into_text(buf.getvalue())
rows = [(b["locator"]["sheet"], b["locator"]["row_start"]) for b in doc.source_blocks]
self.assertEqual(rows, [("明细", 3), ("明细", 5)])
first = doc.source_blocks[0]
self.assertEqual(doc.content[first["start"]:first["end"]], "B: 螺栓\n")