WeKnora/internal/infrastructure/docparser/grpc_parser.go

256 lines
7.0 KiB
Go
Raw Normal View History

package docparser
import (
"context"
"fmt"
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
"io"
"os"
"strconv"
"sync"
"time"
fix(docreader/auth): harden gRPC TLS/Token rollout from #1359 Follow-up to #1359. Addresses a set of correctness and security gaps in the initial docreader auth implementation. - docker-compose: inject GRPC_TLS_*/GRPC_TLS_SERVER_NAME/GRPC_AUTH_TOKEN into the WeKnora-app service. Without this the Go client never saw the knobs, so enabling token auth on the server broke every RPC. - client: bind tokenAuth.RequireTransportSecurity() to TLSEnabled so a bearer token cannot be sent over an insecure channel once TLS is on. - server: load_tls_credentials now raises TLSConfigError on misconfig (cert/key missing, file unreadable, mTLS without CA); main.py exits 1 instead of silently downgrading to insecure. - server: replace endswith("/Check"|"/Watch") health bypass with exact match against /grpc.health.v1.Health/{Check,Watch}. - server: compare tokens with hmac.compare_digest, warn on tokens < 16B. - server: AuthInterceptor now returns an abort handler matching the original RPC kind (unary/stream) and uses context.abort, so streaming RPCs surface UNAUTHENTICATED instead of INTERNAL. - internal/infrastructure/docparser/grpc_parser.go: drop the duplicated TLS/tokenAuth block and reuse docreader/client.LoadAuthConfigFromEnv + BuildDialOptions. Single source of truth for client-side auth. - Add GRPC_TLS_SERVER_NAME (client SNI override) and GRPC_MTLS_REQUIRE_CLIENT_CERT (server explicit mTLS toggle); document the differing CA semantics between client and server in .env*.example. - Reject half-configured client mTLS (cert XOR key) loudly. - Fix missing trailing newline in .env.lite.example. Verified locally: go build ./... and go vet ./... clean; auth.py fail-fast / token paths smoke-tested.
2026-05-16 13:39:10 +00:00
docclient "github.com/Tencent/WeKnora/docreader/client"
"github.com/Tencent/WeKnora/docreader/proto"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/status"
)
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
return size * 1024 * 1024
}
}
return 50 * 1024 * 1024
}
// GRPCDocumentReader implements DocumentReader over gRPC.
type GRPCDocumentReader struct {
mu sync.RWMutex
conn *grpc.ClientConn
client proto.DocReaderClient
addr string
}
func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
p := &GRPCDocumentReader{}
if addr != "" {
if err := p.connect(addr); err != nil {
return nil, err
}
}
return p, nil
}
func (p *GRPCDocumentReader) connect(addr string) error {
fix(docreader/auth): harden gRPC TLS/Token rollout from #1359 Follow-up to #1359. Addresses a set of correctness and security gaps in the initial docreader auth implementation. - docker-compose: inject GRPC_TLS_*/GRPC_TLS_SERVER_NAME/GRPC_AUTH_TOKEN into the WeKnora-app service. Without this the Go client never saw the knobs, so enabling token auth on the server broke every RPC. - client: bind tokenAuth.RequireTransportSecurity() to TLSEnabled so a bearer token cannot be sent over an insecure channel once TLS is on. - server: load_tls_credentials now raises TLSConfigError on misconfig (cert/key missing, file unreadable, mTLS without CA); main.py exits 1 instead of silently downgrading to insecure. - server: replace endswith("/Check"|"/Watch") health bypass with exact match against /grpc.health.v1.Health/{Check,Watch}. - server: compare tokens with hmac.compare_digest, warn on tokens < 16B. - server: AuthInterceptor now returns an abort handler matching the original RPC kind (unary/stream) and uses context.abort, so streaming RPCs surface UNAUTHENTICATED instead of INTERNAL. - internal/infrastructure/docparser/grpc_parser.go: drop the duplicated TLS/tokenAuth block and reuse docreader/client.LoadAuthConfigFromEnv + BuildDialOptions. Single source of truth for client-side auth. - Add GRPC_TLS_SERVER_NAME (client SNI override) and GRPC_MTLS_REQUIRE_CLIENT_CERT (server explicit mTLS toggle); document the differing CA semantics between client and server in .env*.example. - Reject half-configured client mTLS (cert XOR key) loudly. - Fix missing trailing newline in .env.lite.example. Verified locally: go build ./... and go vet ./... clean; auth.py fail-fast / token paths smoke-tested.
2026-05-16 13:39:10 +00:00
authConfig := docclient.LoadAuthConfigFromEnv()
opts, err := authConfig.BuildDialOptions(getMaxMessageSize())
if err != nil {
return fmt.Errorf("failed to build docreader dial options: %w", err)
}
fix(docreader/auth): harden gRPC TLS/Token rollout from #1359 Follow-up to #1359. Addresses a set of correctness and security gaps in the initial docreader auth implementation. - docker-compose: inject GRPC_TLS_*/GRPC_TLS_SERVER_NAME/GRPC_AUTH_TOKEN into the WeKnora-app service. Without this the Go client never saw the knobs, so enabling token auth on the server broke every RPC. - client: bind tokenAuth.RequireTransportSecurity() to TLSEnabled so a bearer token cannot be sent over an insecure channel once TLS is on. - server: load_tls_credentials now raises TLSConfigError on misconfig (cert/key missing, file unreadable, mTLS without CA); main.py exits 1 instead of silently downgrading to insecure. - server: replace endswith("/Check"|"/Watch") health bypass with exact match against /grpc.health.v1.Health/{Check,Watch}. - server: compare tokens with hmac.compare_digest, warn on tokens < 16B. - server: AuthInterceptor now returns an abort handler matching the original RPC kind (unary/stream) and uses context.abort, so streaming RPCs surface UNAUTHENTICATED instead of INTERNAL. - internal/infrastructure/docparser/grpc_parser.go: drop the duplicated TLS/tokenAuth block and reuse docreader/client.LoadAuthConfigFromEnv + BuildDialOptions. Single source of truth for client-side auth. - Add GRPC_TLS_SERVER_NAME (client SNI override) and GRPC_MTLS_REQUIRE_CLIENT_CERT (server explicit mTLS toggle); document the differing CA semantics between client and server in .env*.example. - Reject half-configured client mTLS (cert XOR key) loudly. - Fix missing trailing newline in .env.lite.example. Verified locally: go build ./... and go vet ./... clean; auth.py fail-fast / token paths smoke-tested.
2026-05-16 13:39:10 +00:00
if authConfig.TLSEnabled {
2026-05-16 09:49:31 +00:00
logger.Infof(context.Background(), "TLS enabled for docreader gRPC client")
}
fix(docreader/auth): harden gRPC TLS/Token rollout from #1359 Follow-up to #1359. Addresses a set of correctness and security gaps in the initial docreader auth implementation. - docker-compose: inject GRPC_TLS_*/GRPC_TLS_SERVER_NAME/GRPC_AUTH_TOKEN into the WeKnora-app service. Without this the Go client never saw the knobs, so enabling token auth on the server broke every RPC. - client: bind tokenAuth.RequireTransportSecurity() to TLSEnabled so a bearer token cannot be sent over an insecure channel once TLS is on. - server: load_tls_credentials now raises TLSConfigError on misconfig (cert/key missing, file unreadable, mTLS without CA); main.py exits 1 instead of silently downgrading to insecure. - server: replace endswith("/Check"|"/Watch") health bypass with exact match against /grpc.health.v1.Health/{Check,Watch}. - server: compare tokens with hmac.compare_digest, warn on tokens < 16B. - server: AuthInterceptor now returns an abort handler matching the original RPC kind (unary/stream) and uses context.abort, so streaming RPCs surface UNAUTHENTICATED instead of INTERNAL. - internal/infrastructure/docparser/grpc_parser.go: drop the duplicated TLS/tokenAuth block and reuse docreader/client.LoadAuthConfigFromEnv + BuildDialOptions. Single source of truth for client-side auth. - Add GRPC_TLS_SERVER_NAME (client SNI override) and GRPC_MTLS_REQUIRE_CLIENT_CERT (server explicit mTLS toggle); document the differing CA semantics between client and server in .env*.example. - Reject half-configured client mTLS (cert XOR key) loudly. - Fix missing trailing newline in .env.lite.example. Verified locally: go build ./... and go vet ./... clean; auth.py fail-fast / token paths smoke-tested.
2026-05-16 13:39:10 +00:00
if authConfig.AuthToken != "" {
logger.Infof(context.Background(),
"Token authentication enabled for docreader gRPC client (TLS=%v)",
authConfig.TLSEnabled,
)
2026-05-16 09:49:31 +00:00
}
resolver.SetDefaultScheme("dns")
start := time.Now()
conn, err := grpc.Dial("dns:///"+addr, opts...)
if err != nil {
return fmt.Errorf("failed to connect to docreader: %w", err)
}
logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start))
p.conn = conn
p.client = proto.NewDocReaderClient(conn)
p.addr = addr
return nil
}
func (p *GRPCDocumentReader) Reconnect(addr string) error {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil {
_ = p.conn.Close()
p.conn = nil
p.client = nil
p.addr = ""
}
return p.connect(addr)
}
func (p *GRPCDocumentReader) IsConnected() bool {
p.mu.RLock()
defer p.mu.RUnlock()
return p.conn != nil
}
func (p *GRPCDocumentReader) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil {
return p.conn.Close()
}
return nil
}
var errNotConnected = fmt.Errorf("docreader service not connected")
func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
p.mu.RLock()
client := p.client
p.mu.RUnlock()
if client == nil {
return nil, errNotConnected
}
protoReq := &proto.ReadRequest{
FileContent: req.FileContent,
FileName: req.FileName,
FileType: req.FileType,
Url: req.URL,
Title: req.Title,
RequestId: req.RequestID,
Config: &proto.ReadConfig{
ParserEngine: req.ParserEngine,
ParserEngineOverrides: req.ParserEngineOverrides,
},
}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
// Use the streaming RPC so documents with many page images (large scanned
// PDFs) are not capped by the unary message-size limit. The meta frame
// arrives first, followed by one frame per image.
result, err := p.readStream(ctx, client, protoReq)
if err != nil {
// An older docreader build may not implement ReadStream. Fall back to
// the unary Read RPC so a version-skewed deployment still parses
// documents (small/medium docs only — the unary path remains capped by
// the gRPC message-size limit, which is exactly what streaming avoids).
if status.Code(err) == codes.Unimplemented {
logger.Warnf(ctx, "docreader ReadStream unimplemented, falling back to unary Read: %v", err)
return p.readUnary(ctx, client, protoReq)
}
return nil, err
}
return result, nil
}
// readStream consumes the server-streaming ReadStream RPC: one meta frame
// followed by one frame per image. Errors are returned verbatim so the caller
// can inspect the gRPC status code (e.g. Unimplemented) for fallback.
func (p *GRPCDocumentReader) readStream(
ctx context.Context, client proto.DocReaderClient, protoReq *proto.ReadRequest,
) (*types.ReadResult, error) {
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
stream, err := client.ReadStream(ctx, protoReq)
if err != nil {
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
return nil, fmt.Errorf("gRPC ReadStream failed: %w", err)
}
feat(docreader): speed up scanned-PDF parsing, stream image results, isolate heavy async queues Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end. This change addresses the parse, transport, and task-scheduling layers: docreader (parse + transport): - Parallelize per-page scanned rendering across processes (forkserver/fork), with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not thread-safe so we fan out across processes. Configurable via DOCREADER_PDF_RENDER_PARALLELISM. - Add server-streaming ReadStream RPC: emit one meta frame then one frame per image, so documents with many page images are no longer capped by the unary gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over the 50MB cap) and memory is bounded on both ends. Unary Read is kept for backward compatibility; the Go production reader switches to ReadStream. VLM: - Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise the default 90s -> 180s so dense scanned-page OCR does not time out with "context deadline exceeded". Async task queues: - Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so a single large document cannot saturate the shared worker pool and block user-facing document parsing: image:multimodal -> "multimodal" chunk:extract -> "graph" question:generation -> "question" - Register the new queues in the server weight map and the cancel inspector's scanned-queue set (so cancelling a knowledge still purges its pending tasks).
2026-06-02 01:50:09 +00:00
result := &types.ReadResult{}
gotMeta := false
for {
frame, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
return nil, fmt.Errorf("gRPC ReadStream recv failed: %w", recvErr)
}
if meta := frame.GetMeta(); meta != nil {
gotMeta = true
result.MarkdownContent = meta.GetMarkdownContent()
result.ImageDirPath = meta.GetImageDirPath()
result.Metadata = meta.GetMetadata()
result.Error = meta.GetError()
if n := meta.GetImageCount(); n > 0 {
result.ImageRefs = make([]types.ImageRef, 0, n)
}
continue
}
if img := frame.GetImage(); img != nil {
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
Filename: img.GetFilename(),
OriginalRef: img.GetOriginalRef(),
MimeType: img.GetMimeType(),
StorageKey: img.GetStorageKey(),
ImageData: img.GetImageData(),
})
}
}
if !gotMeta {
return nil, fmt.Errorf("gRPC ReadStream returned no metadata frame")
}
return result, nil
}
// readUnary calls the legacy unary Read RPC. Used only as a compatibility
// fallback when the connected docreader does not implement ReadStream.
func (p *GRPCDocumentReader) readUnary(
ctx context.Context, client proto.DocReaderClient, protoReq *proto.ReadRequest,
) (*types.ReadResult, error) {
resp, err := client.Read(ctx, protoReq)
if err != nil {
return nil, fmt.Errorf("gRPC Read failed: %w", err)
}
result := &types.ReadResult{
MarkdownContent: resp.GetMarkdownContent(),
ImageDirPath: resp.GetImageDirPath(),
Metadata: resp.GetMetadata(),
Error: resp.GetError(),
}
if refs := resp.GetImageRefs(); len(refs) > 0 {
result.ImageRefs = make([]types.ImageRef, 0, len(refs))
for _, img := range refs {
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
Filename: img.GetFilename(),
OriginalRef: img.GetOriginalRef(),
MimeType: img.GetMimeType(),
StorageKey: img.GetStorageKey(),
ImageData: img.GetImageData(),
})
}
}
return result, nil
}
func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
p.mu.RLock()
client := p.client
p.mu.RUnlock()
if client == nil {
return nil, errNotConnected
}
resp, err := client.ListEngines(ctx, &proto.ListEnginesRequest{ConfigOverrides: overrides})
if err != nil {
return nil, fmt.Errorf("gRPC ListEngines failed: %w", err)
}
result := make([]types.ParserEngineInfo, 0, len(resp.GetEngines()))
for _, e := range resp.GetEngines() {
result = append(result, types.ParserEngineInfo{
Name: e.GetName(),
Description: e.GetDescription(),
FileTypes: e.GetFileTypes(),
Available: e.GetAvailable(),
UnavailableReason: e.GetUnavailableReason(),
})
}
return result, nil
}