WeKnora/docreader/tests/test_endecode.py

61 lines
2.3 KiB
Python
Raw Normal View History

fix(docreader): stop a non-ASCII base64 payload from failing the whole parse encode_image documents errors="ignore" as "return empty bytes on decoding errors", but it only catches binascii.Error. base64.b64decode raises that for bad padding or a bad alphabet; for a str that is not pure ASCII it fails in _bytes_from_decode_data first and raises a plain ValueError, which escapes. The one caller that passes errors="ignore" is the markdown base64-image extractor, whose payload group is `([^\)]+)` — everything up to the closing paren, not the base64 alphabet. A CJK image title therefore lands inside the payload: ![x](data:image/png;base64,iVBORw0KGgo= "图片") ^^^^ captured as base64 MarkdownImageBase64.parse_into_text does not catch, PipelineParser does not catch, and BaseParser.parse does not catch, so the ValueError reaches the gRPC handler and the whole document comes back as ReadResponse(error="string argument should contain only ASCII characters"). One image title fails the entire upload, and the message names nothing the user can act on. The same title written in ASCII decodes fine, because b64decode with validate=False discards the quotes and spaces. The failure only shows up for titles in the languages this project is mostly used in. Catching ValueError covers both cases — binascii.Error subclasses it — so the undecodable image is skipped and logged, which is what the extractor's own `if not image_byte` branch already expects to happen. Not addressed here: the pattern swallowing the title is a separate bug, and narrowing that group changes which markdown the extractor matches. Filed as its own change rather than folded in.
2026-09-21 14:25:25 +00:00
import base64
import binascii
import re
import unittest
from docreader.utils.endecode import encode_image
# One valid PNG signature, the shortest payload that proves a decode happened.
PNG_SIGNATURE = bytes.fromhex("89504e470d0a1a0a")
VALID_B64 = base64.b64encode(PNG_SIGNATURE).decode()
class TestEncodeImage(unittest.TestCase):
def test_valid_base64_round_trips(self):
self.assertEqual(encode_image(VALID_B64), PNG_SIGNATURE)
def test_ignore_swallows_bad_padding(self):
"""A truncated payload raises binascii.Error, which 'ignore' absorbs."""
self.assertEqual(encode_image(VALID_B64[:-3], errors="ignore"), b"")
def test_ignore_swallows_non_ascii(self):
"""Non-ASCII input is a decoding error too, so 'ignore' must absorb it.
base64.b64decode rejects a non-ASCII str with a plain ValueError rather
than binascii.Error, so catching only binascii.Error lets it escape a
call that asked not to raise.
"""
self.assertEqual(encode_image(VALID_B64 + ' "图片"', errors="ignore"), b"")
def test_strict_still_raises_on_non_ascii(self):
with self.assertRaises(ValueError):
encode_image(VALID_B64 + ' "图片"')
def test_strict_still_raises_on_bad_padding(self):
with self.assertRaises(binascii.Error):
encode_image(VALID_B64[:-3])
class TestMarkdownBase64Reachability(unittest.TestCase):
"""The payload group of the markdown data-URI pattern is not ASCII-only.
Pinned here rather than in the parser tests because importing
docreader.parser pulls in the whole parser dependency set; the pattern
itself is what decides whether encode_image can ever see non-ASCII.
"""
# docreader/parser/markdown_parser.py
B64_PATTERN = re.compile(r"!\[(.*?)\]\(data:image/([^;]+);base64,([^\)]+)\)")
def test_markdown_title_lands_in_the_payload_group(self):
markdown = '![x](data:image/png;base64,%s "图片")' % VALID_B64
match = self.B64_PATTERN.search(markdown)
self.assertIsNotNone(match)
self.assertIn("图片", match.group(3))
# What the parser then hands to encode_image(errors="ignore").
self.assertEqual(encode_image(match.group(3), errors="ignore"), b"")
if __name__ == "__main__":
unittest.main()