WeKnora/docreader/proto/docreader.pb.go

887 lines
28 KiB
Go
Raw Normal View History

2025-08-05 07:08:07 +00:00
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
// protoc-gen-go v1.36.6
// protoc v6.33.4
2025-08-05 07:08:07 +00:00
// source: docreader.proto
package proto
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type ReadConfig struct {
state protoimpl.MessageState `protogen:"open.v1"`
ParserEngine string `protobuf:"bytes,1,opt,name=parser_engine,json=parserEngine,proto3" json:"parser_engine,omitempty"`
ParserEngineOverrides map[string]string `protobuf:"bytes,2,rep,name=parser_engine_overrides,json=parserEngineOverrides,proto3" json:"parser_engine_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
2025-08-13 13:56:20 +00:00
}
func (x *ReadConfig) Reset() {
*x = ReadConfig{}
mi := &file_docreader_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReadConfig) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReadConfig) ProtoMessage() {}
func (x *ReadConfig) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReadConfig.ProtoReflect.Descriptor instead.
func (*ReadConfig) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{0}
}
func (x *ReadConfig) GetParserEngine() string {
if x != nil {
return x.ParserEngine
}
return ""
}
func (x *ReadConfig) GetParserEngineOverrides() map[string]string {
if x != nil {
return x.ParserEngineOverrides
}
return nil
}
// Unified read request: set file_content for file mode, url for URL mode.
type ReadRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
FileName string `protobuf:"bytes,2,opt,name=file_name,json=fileName,proto3" json:"file_name,omitempty"`
FileType string `protobuf:"bytes,3,opt,name=file_type,json=fileType,proto3" json:"file_type,omitempty"`
Url string `protobuf:"bytes,4,opt,name=url,proto3" json:"url,omitempty"`
Title string `protobuf:"bytes,5,opt,name=title,proto3" json:"title,omitempty"`
Config *ReadConfig `protobuf:"bytes,6,opt,name=config,proto3" json:"config,omitempty"`
RequestId string `protobuf:"bytes,7,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReadRequest) Reset() {
*x = ReadRequest{}
mi := &file_docreader_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReadRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReadRequest) ProtoMessage() {}
func (x *ReadRequest) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReadRequest.ProtoReflect.Descriptor instead.
func (*ReadRequest) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{1}
}
func (x *ReadRequest) GetFileContent() []byte {
if x != nil {
return x.FileContent
}
return nil
}
func (x *ReadRequest) GetFileName() string {
if x != nil {
return x.FileName
}
return ""
}
func (x *ReadRequest) GetFileType() string {
if x != nil {
return x.FileType
}
return ""
}
func (x *ReadRequest) GetUrl() string {
if x != nil {
return x.Url
}
return ""
}
func (x *ReadRequest) GetTitle() string {
if x != nil {
return x.Title
}
return ""
}
func (x *ReadRequest) GetConfig() *ReadConfig {
if x != nil {
return x.Config
}
return nil
2025-08-05 07:08:07 +00:00
}
func (x *ReadRequest) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
type ImageRef struct {
state protoimpl.MessageState `protogen:"open.v1"`
Filename string `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"`
OriginalRef string `protobuf:"bytes,2,opt,name=original_ref,json=originalRef,proto3" json:"original_ref,omitempty"`
MimeType string `protobuf:"bytes,3,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"`
StorageKey string `protobuf:"bytes,4,opt,name=storage_key,json=storageKey,proto3" json:"storage_key,omitempty"` // download URL from shared storage
ImageData []byte `protobuf:"bytes,5,opt,name=image_data,json=imageData,proto3" json:"image_data,omitempty"` // inline bytes fallback
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ImageRef) Reset() {
*x = ImageRef{}
mi := &file_docreader_proto_msgTypes[2]
2025-08-05 07:08:07 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ImageRef) String() string {
2025-08-05 07:08:07 +00:00
return protoimpl.X.MessageStringOf(x)
}
func (*ImageRef) ProtoMessage() {}
2025-08-05 07:08:07 +00:00
func (x *ImageRef) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[2]
2025-08-05 07:08:07 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ImageRef.ProtoReflect.Descriptor instead.
func (*ImageRef) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{2}
2025-08-05 07:08:07 +00:00
}
func (x *ImageRef) GetFilename() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Filename
2025-08-05 07:08:07 +00:00
}
return ""
2025-08-05 07:08:07 +00:00
}
func (x *ImageRef) GetOriginalRef() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.OriginalRef
2025-08-05 07:08:07 +00:00
}
return ""
2025-08-05 07:08:07 +00:00
}
func (x *ImageRef) GetMimeType() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.MimeType
2025-08-05 07:08:07 +00:00
}
return ""
2025-08-05 07:08:07 +00:00
}
func (x *ImageRef) GetStorageKey() string {
if x != nil {
return x.StorageKey
}
return ""
}
func (x *ImageRef) GetImageData() []byte {
if x != nil {
return x.ImageData
}
return nil
}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
// SourceBlock ties a range of markdown_content to the place in the original
// file it was parsed from, so a citation can open the file there.
type SourceBlock struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Unicode code point offsets into markdown_content, end exclusive.
Start uint32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"`
End uint32 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
// JSON-encoded locator: {"type":"pdf","page":3,"bbox":[x0,y0,x1,y1]}
// with ordinals 1-based and bbox as fractions of the page, origin top-left.
LocatorJson string `protobuf:"bytes,3,opt,name=locator_json,json=locatorJson,proto3" json:"locator_json,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *SourceBlock) Reset() {
*x = SourceBlock{}
mi := &file_docreader_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *SourceBlock) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SourceBlock) ProtoMessage() {}
func (x *SourceBlock) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[3]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SourceBlock.ProtoReflect.Descriptor instead.
func (*SourceBlock) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{3}
}
func (x *SourceBlock) GetStart() uint32 {
if x != nil {
return x.Start
}
return 0
}
func (x *SourceBlock) GetEnd() uint32 {
if x != nil {
return x.End
}
return 0
}
func (x *SourceBlock) GetLocatorJson() string {
if x != nil {
return x.LocatorJson
}
return ""
}
type ReadResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
MarkdownContent string `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"`
ImageRefs []*ImageRef `protobuf:"bytes,2,rep,name=image_refs,json=imageRefs,proto3" json:"image_refs,omitempty"`
ImageDirPath string `protobuf:"bytes,3,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"`
Metadata map[string]string `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
SourceBlocks []*SourceBlock `protobuf:"bytes,6,rep,name=source_blocks,json=sourceBlocks,proto3" json:"source_blocks,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
2025-08-05 07:08:07 +00:00
}
func (x *ReadResponse) Reset() {
*x = ReadResponse{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[4]
2025-08-05 07:08:07 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReadResponse) String() string {
2025-08-05 07:08:07 +00:00
return protoimpl.X.MessageStringOf(x)
}
func (*ReadResponse) ProtoMessage() {}
2025-08-05 07:08:07 +00:00
func (x *ReadResponse) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[4]
2025-08-05 07:08:07 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead.
func (*ReadResponse) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{4}
2025-08-05 07:08:07 +00:00
}
func (x *ReadResponse) GetMarkdownContent() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.MarkdownContent
2025-08-05 07:08:07 +00:00
}
return ""
2025-08-05 07:08:07 +00:00
}
func (x *ReadResponse) GetImageRefs() []*ImageRef {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.ImageRefs
2025-08-05 07:08:07 +00:00
}
return nil
2025-08-05 07:08:07 +00:00
}
func (x *ReadResponse) GetImageDirPath() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.ImageDirPath
2025-08-05 07:08:07 +00:00
}
return ""
}
func (x *ReadResponse) GetMetadata() map[string]string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Metadata
2025-08-05 07:08:07 +00:00
}
return nil
}
func (x *ReadResponse) GetError() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Error
2025-08-05 07:08:07 +00:00
}
return ""
}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
func (x *ReadResponse) GetSourceBlocks() []*SourceBlock {
if x != nil {
return x.SourceBlocks
}
return nil
}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
// Metadata header for a streamed read. Sent exactly once, before any image.
type ReadStreamMeta struct {
state protoimpl.MessageState `protogen:"open.v1"`
MarkdownContent string `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"`
ImageDirPath string `protobuf:"bytes,2,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"`
Metadata map[string]string `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
ImageCount uint32 `protobuf:"varint,5,opt,name=image_count,json=imageCount,proto3" json:"image_count,omitempty"` // best-effort total image count (0 if unknown)
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
SourceBlocks []*SourceBlock `protobuf:"bytes,6,rep,name=source_blocks,json=sourceBlocks,proto3" json:"source_blocks,omitempty"`
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReadStreamMeta) Reset() {
*x = ReadStreamMeta{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[5]
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReadStreamMeta) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReadStreamMeta) ProtoMessage() {}
func (x *ReadStreamMeta) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[5]
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReadStreamMeta.ProtoReflect.Descriptor instead.
func (*ReadStreamMeta) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{5}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
}
func (x *ReadStreamMeta) GetMarkdownContent() string {
if x != nil {
return x.MarkdownContent
}
return ""
}
func (x *ReadStreamMeta) GetImageDirPath() string {
if x != nil {
return x.ImageDirPath
}
return ""
}
func (x *ReadStreamMeta) GetMetadata() map[string]string {
if x != nil {
return x.Metadata
}
return nil
}
func (x *ReadStreamMeta) GetError() string {
if x != nil {
return x.Error
}
return ""
}
func (x *ReadStreamMeta) GetImageCount() uint32 {
if x != nil {
return x.ImageCount
}
return 0
}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
func (x *ReadStreamMeta) GetSourceBlocks() []*SourceBlock {
if x != nil {
return x.SourceBlocks
}
return nil
}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
// One frame of a ReadStream. The first frame carries `meta`; every subsequent
// frame carries a single `image`.
type ReadStreamResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Types that are valid to be assigned to Payload:
//
// *ReadStreamResponse_Meta
// *ReadStreamResponse_Image
Payload isReadStreamResponse_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReadStreamResponse) Reset() {
*x = ReadStreamResponse{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[6]
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReadStreamResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReadStreamResponse) ProtoMessage() {}
func (x *ReadStreamResponse) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[6]
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReadStreamResponse.ProtoReflect.Descriptor instead.
func (*ReadStreamResponse) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{6}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
}
func (x *ReadStreamResponse) GetPayload() isReadStreamResponse_Payload {
if x != nil {
return x.Payload
}
return nil
}
func (x *ReadStreamResponse) GetMeta() *ReadStreamMeta {
if x != nil {
if x, ok := x.Payload.(*ReadStreamResponse_Meta); ok {
return x.Meta
}
}
return nil
}
func (x *ReadStreamResponse) GetImage() *ImageRef {
if x != nil {
if x, ok := x.Payload.(*ReadStreamResponse_Image); ok {
return x.Image
}
}
return nil
}
type isReadStreamResponse_Payload interface {
isReadStreamResponse_Payload()
}
type ReadStreamResponse_Meta struct {
Meta *ReadStreamMeta `protobuf:"bytes,1,opt,name=meta,proto3,oneof"`
}
type ReadStreamResponse_Image struct {
Image *ImageRef `protobuf:"bytes,2,opt,name=image,proto3,oneof"`
}
func (*ReadStreamResponse_Meta) isReadStreamResponse_Payload() {}
func (*ReadStreamResponse_Image) isReadStreamResponse_Payload() {}
type ListEnginesRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
ConfigOverrides map[string]string `protobuf:"bytes,1,rep,name=config_overrides,json=configOverrides,proto3" json:"config_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
2025-08-05 07:08:07 +00:00
}
func (x *ListEnginesRequest) Reset() {
*x = ListEnginesRequest{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[7]
2025-08-05 07:08:07 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ListEnginesRequest) String() string {
2025-08-05 07:08:07 +00:00
return protoimpl.X.MessageStringOf(x)
}
func (*ListEnginesRequest) ProtoMessage() {}
2025-08-05 07:08:07 +00:00
func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[7]
2025-08-05 07:08:07 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ListEnginesRequest.ProtoReflect.Descriptor instead.
func (*ListEnginesRequest) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{7}
2025-08-05 07:08:07 +00:00
}
func (x *ListEnginesRequest) GetConfigOverrides() map[string]string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.ConfigOverrides
2025-08-05 07:08:07 +00:00
}
return nil
}
type ParserEngineInfo struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"`
FileTypes []string `protobuf:"bytes,3,rep,name=file_types,json=fileTypes,proto3" json:"file_types,omitempty"`
Available bool `protobuf:"varint,4,opt,name=available,proto3" json:"available,omitempty"`
UnavailableReason string `protobuf:"bytes,5,opt,name=unavailable_reason,json=unavailableReason,proto3" json:"unavailable_reason,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
2025-08-05 07:08:07 +00:00
}
func (x *ParserEngineInfo) Reset() {
*x = ParserEngineInfo{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[8]
2025-08-05 07:08:07 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ParserEngineInfo) String() string {
2025-08-05 07:08:07 +00:00
return protoimpl.X.MessageStringOf(x)
}
func (*ParserEngineInfo) ProtoMessage() {}
2025-08-05 07:08:07 +00:00
func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[8]
2025-08-05 07:08:07 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ParserEngineInfo.ProtoReflect.Descriptor instead.
func (*ParserEngineInfo) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{8}
2025-08-05 07:08:07 +00:00
}
func (x *ParserEngineInfo) GetName() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Name
2025-08-05 07:08:07 +00:00
}
return ""
}
func (x *ParserEngineInfo) GetDescription() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Description
2025-08-05 07:08:07 +00:00
}
return ""
}
func (x *ParserEngineInfo) GetFileTypes() []string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.FileTypes
2025-08-05 07:08:07 +00:00
}
return nil
2025-08-05 07:08:07 +00:00
}
func (x *ParserEngineInfo) GetAvailable() bool {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Available
2025-08-05 07:08:07 +00:00
}
return false
2025-08-05 07:08:07 +00:00
}
func (x *ParserEngineInfo) GetUnavailableReason() string {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.UnavailableReason
2025-08-05 07:08:07 +00:00
}
return ""
2025-08-05 07:08:07 +00:00
}
type ListEnginesResponse struct {
2025-08-05 07:08:07 +00:00
state protoimpl.MessageState `protogen:"open.v1"`
Engines []*ParserEngineInfo `protobuf:"bytes,1,rep,name=engines,proto3" json:"engines,omitempty"`
2025-08-05 07:08:07 +00:00
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ListEnginesResponse) Reset() {
*x = ListEnginesResponse{}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[9]
2025-08-05 07:08:07 +00:00
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ListEnginesResponse) String() string {
2025-08-05 07:08:07 +00:00
return protoimpl.X.MessageStringOf(x)
}
func (*ListEnginesResponse) ProtoMessage() {}
2025-08-05 07:08:07 +00:00
func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
mi := &file_docreader_proto_msgTypes[9]
2025-08-05 07:08:07 +00:00
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ListEnginesResponse.ProtoReflect.Descriptor instead.
func (*ListEnginesResponse) Descriptor() ([]byte, []int) {
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
return file_docreader_proto_rawDescGZIP(), []int{9}
2025-08-05 07:08:07 +00:00
}
func (x *ListEnginesResponse) GetEngines() []*ParserEngineInfo {
2025-08-05 07:08:07 +00:00
if x != nil {
return x.Engines
2025-08-05 07:08:07 +00:00
}
return nil
}
2025-08-05 07:08:07 +00:00
var File_docreader_proto protoreflect.FileDescriptor
const file_docreader_proto_rawDesc = "" +
"\n" +
"\x0fdocreader.proto\x12\tdocreader\"\xeb\x01\n" +
"\n" +
"ReadConfig\x12#\n" +
"\rparser_engine\x18\x01 \x01(\tR\fparserEngine\x12h\n" +
"\x17parser_engine_overrides\x18\x02 \x03(\v20.docreader.ReadConfig.ParserEngineOverridesEntryR\x15parserEngineOverrides\x1aH\n" +
"\x1aParserEngineOverridesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\x03\x10\x04\"\xe0\x01\n" +
"\vReadRequest\x12!\n" +
"\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" +
"\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" +
"\tfile_type\x18\x03 \x01(\tR\bfileType\x12\x10\n" +
"\x03url\x18\x04 \x01(\tR\x03url\x12\x14\n" +
"\x05title\x18\x05 \x01(\tR\x05title\x12-\n" +
"\x06config\x18\x06 \x01(\v2\x15.docreader.ReadConfigR\x06config\x12\x1d\n" +
2025-08-05 07:08:07 +00:00
"\n" +
"request_id\x18\a \x01(\tR\trequestId\"\xa6\x01\n" +
"\bImageRef\x12\x1a\n" +
"\bfilename\x18\x01 \x01(\tR\bfilename\x12!\n" +
"\foriginal_ref\x18\x02 \x01(\tR\voriginalRef\x12\x1b\n" +
"\tmime_type\x18\x03 \x01(\tR\bmimeType\x12\x1f\n" +
"\vstorage_key\x18\x04 \x01(\tR\n" +
"storageKey\x12\x1d\n" +
2025-08-05 07:08:07 +00:00
"\n" +
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
"image_data\x18\x05 \x01(\fR\timageData\"X\n" +
"\vSourceBlock\x12\x14\n" +
"\x05start\x18\x01 \x01(\rR\x05start\x12\x10\n" +
"\x03end\x18\x02 \x01(\rR\x03end\x12!\n" +
"\flocator_json\x18\x03 \x01(\tR\vlocatorJson\"\xe6\x02\n" +
"\fReadResponse\x12)\n" +
"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x122\n" +
2025-08-05 07:08:07 +00:00
"\n" +
"image_refs\x18\x02 \x03(\v2\x13.docreader.ImageRefR\timageRefs\x12$\n" +
"\x0eimage_dir_path\x18\x03 \x01(\tR\fimageDirPath\x12A\n" +
"\bmetadata\x18\x04 \x03(\v2%.docreader.ReadResponse.MetadataEntryR\bmetadata\x12\x14\n" +
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
"\x05error\x18\x05 \x01(\tR\x05error\x12;\n" +
"\rsource_blocks\x18\x06 \x03(\v2\x16.docreader.SourceBlockR\fsourceBlocks\x1a;\n" +
"\rMetadataEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xd7\x02\n" +
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
"\x0eReadStreamMeta\x12)\n" +
"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x12$\n" +
"\x0eimage_dir_path\x18\x02 \x01(\tR\fimageDirPath\x12C\n" +
"\bmetadata\x18\x03 \x03(\v2'.docreader.ReadStreamMeta.MetadataEntryR\bmetadata\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error\x12\x1f\n" +
"\vimage_count\x18\x05 \x01(\rR\n" +
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
"imageCount\x12;\n" +
"\rsource_blocks\x18\x06 \x03(\v2\x16.docreader.SourceBlockR\fsourceBlocks\x1a;\n" +
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
"\rMetadataEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"}\n" +
"\x12ReadStreamResponse\x12/\n" +
"\x04meta\x18\x01 \x01(\v2\x19.docreader.ReadStreamMetaH\x00R\x04meta\x12+\n" +
"\x05image\x18\x02 \x01(\v2\x13.docreader.ImageRefH\x00R\x05imageB\t\n" +
"\apayload\"\xb7\x01\n" +
"\x12ListEnginesRequest\x12]\n" +
"\x10config_overrides\x18\x01 \x03(\v22.docreader.ListEnginesRequest.ConfigOverridesEntryR\x0fconfigOverrides\x1aB\n" +
"\x14ConfigOverridesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb4\x01\n" +
"\x10ParserEngineInfo\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12 \n" +
"\vdescription\x18\x02 \x01(\tR\vdescription\x12\x1d\n" +
"\n" +
"file_types\x18\x03 \x03(\tR\tfileTypes\x12\x1c\n" +
"\tavailable\x18\x04 \x01(\bR\tavailable\x12-\n" +
"\x12unavailable_reason\x18\x05 \x01(\tR\x11unavailableReason\"L\n" +
"\x13ListEnginesResponse\x125\n" +
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
"\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines2\xdf\x01\n" +
"\tDocReader\x129\n" +
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
"\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" +
"\n" +
"ReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x000\x01\x12N\n" +
"\vListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
2025-08-05 07:08:07 +00:00
var (
file_docreader_proto_rawDescOnce sync.Once
file_docreader_proto_rawDescData []byte
)
func file_docreader_proto_rawDescGZIP() []byte {
file_docreader_proto_rawDescOnce.Do(func() {
file_docreader_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)))
})
return file_docreader_proto_rawDescData
}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
2025-08-05 07:08:07 +00:00
var file_docreader_proto_goTypes = []any{
(*ReadConfig)(nil), // 0: docreader.ReadConfig
(*ReadRequest)(nil), // 1: docreader.ReadRequest
(*ImageRef)(nil), // 2: docreader.ImageRef
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
(*SourceBlock)(nil), // 3: docreader.SourceBlock
(*ReadResponse)(nil), // 4: docreader.ReadResponse
(*ReadStreamMeta)(nil), // 5: docreader.ReadStreamMeta
(*ReadStreamResponse)(nil), // 6: docreader.ReadStreamResponse
(*ListEnginesRequest)(nil), // 7: docreader.ListEnginesRequest
(*ParserEngineInfo)(nil), // 8: docreader.ParserEngineInfo
(*ListEnginesResponse)(nil), // 9: docreader.ListEnginesResponse
nil, // 10: docreader.ReadConfig.ParserEngineOverridesEntry
nil, // 11: docreader.ReadResponse.MetadataEntry
nil, // 12: docreader.ReadStreamMeta.MetadataEntry
nil, // 13: docreader.ListEnginesRequest.ConfigOverridesEntry
2025-08-05 07:08:07 +00:00
}
var file_docreader_proto_depIdxs = []int32{
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
10, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
0, // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig
2, // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
11, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
3, // 4: docreader.ReadResponse.source_blocks:type_name -> docreader.SourceBlock
12, // 5: docreader.ReadStreamMeta.metadata:type_name -> docreader.ReadStreamMeta.MetadataEntry
3, // 6: docreader.ReadStreamMeta.source_blocks:type_name -> docreader.SourceBlock
5, // 7: docreader.ReadStreamResponse.meta:type_name -> docreader.ReadStreamMeta
2, // 8: docreader.ReadStreamResponse.image:type_name -> docreader.ImageRef
13, // 9: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
8, // 10: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
1, // 11: docreader.DocReader.Read:input_type -> docreader.ReadRequest
1, // 12: docreader.DocReader.ReadStream:input_type -> docreader.ReadRequest
7, // 13: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
4, // 14: docreader.DocReader.Read:output_type -> docreader.ReadResponse
6, // 15: docreader.DocReader.ReadStream:output_type -> docreader.ReadStreamResponse
9, // 16: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
14, // [14:17] is the sub-list for method output_type
11, // [11:14] is the sub-list for method input_type
11, // [11:11] is the sub-list for extension type_name
11, // [11:11] is the sub-list for extension extendee
0, // [0:11] is the sub-list for field type_name
2025-08-05 07:08:07 +00:00
}
func init() { file_docreader_proto_init() }
func file_docreader_proto_init() {
if File_docreader_proto != nil {
return
}
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
file_docreader_proto_msgTypes[6].OneofWrappers = []any{
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
(*ReadStreamResponse_Meta)(nil),
(*ReadStreamResponse_Image)(nil),
}
2025-08-05 07:08:07 +00:00
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
NumEnums: 0,
feat(chat): open cited documents at the cited passage (#3700) * feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
2026-09-25 05:26:16 +00:00
NumMessages: 14,
2025-08-05 07:08:07 +00:00
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_docreader_proto_goTypes,
DependencyIndexes: file_docreader_proto_depIdxs,
MessageInfos: file_docreader_proto_msgTypes,
}.Build()
File_docreader_proto = out.File
file_docreader_proto_goTypes = nil
file_docreader_proto_depIdxs = nil
}