feat(docparser): 引入 anydoc 引擎,在 Go 进程内解析 office 文档
通过 cgo 链接 anydoc Rust 静态库,把 docx/pptx/xlsx 等转为带原位图片的 Markdown;扫描版 PDF 无文字层时回落到 DocReader OCR。默认构建不链接该库。 Co-authored-by: lyingbug <lyingbug@users.noreply.github.com>main
parent
b18ae0f734
commit
e123f8247e
|
|
@ -5,7 +5,7 @@ tmp_dir = "tmp"
|
|||
[build]
|
||||
args_bin = []
|
||||
bin = "./tmp/main"
|
||||
cmd = "go build -ldflags=\"$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'\" -o ./tmp/main ./cmd/server"
|
||||
cmd = "go build -tags \"${GO_BUILD_TAGS}\" -ldflags=\"$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'\" -o ./tmp/main ./cmd/server"
|
||||
delay = 1000
|
||||
exclude_dir = ["assets", "tmp", "vendor", "testdata", "frontend", "migrations", "node_modules", "docs", "logs", "data", "WeKnora-Chrome-Extension", "Formula", "dataset", "docreader", "docker", "llm_debug", "web", "mcp-server", "misc", "scripts", "dist", "client", "helm", "website-docs"]
|
||||
exclude_file = []
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ on:
|
|||
- "scripts/**"
|
||||
- "skills/preloaded/**"
|
||||
- "docker/Dockerfile.app"
|
||||
- "third_party/anydoc-go/**"
|
||||
- ".github/workflows/app.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
|
|
@ -27,6 +28,7 @@ on:
|
|||
- "scripts/**"
|
||||
- "skills/preloaded/**"
|
||||
- "docker/Dockerfile.app"
|
||||
- "third_party/anydoc-go/**"
|
||||
- ".github/workflows/app.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
|
|
@ -96,3 +98,49 @@ jobs:
|
|||
|
||||
- name: Build server
|
||||
run: go build ./cmd/server
|
||||
|
||||
anydoc:
|
||||
name: anydoc parser engine
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: true
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo-audit
|
||||
id: cargo-audit-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cargo/bin/cargo-audit
|
||||
key: cargo-audit-${{ runner.os }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
if: steps.cargo-audit-cache.outputs.cache-hit != 'true'
|
||||
run: cargo install cargo-audit --locked
|
||||
|
||||
# The vendored bindings pull anydoc and its PDF stack from crates.io, and
|
||||
# WeKnora parses untrusted uploads in-process. A Rust advisory there (as
|
||||
# in RUSTSEC-2026-0187, an uncatchable abort on a hostile PDF) has to
|
||||
# break the build, not wait for someone to notice. Vulnerabilities fail
|
||||
# the job; unmaintained-crate warnings only report, since a transitive
|
||||
# crate going unmaintained is not something this repo can act on.
|
||||
- name: Audit the vendored dependency tree
|
||||
working-directory: third_party/anydoc-go
|
||||
run: cargo audit
|
||||
|
||||
# The archive is a build artifact, not a committed blob, so CI is what
|
||||
# keeps the vendored bindings in third_party/anydoc-go compiling.
|
||||
- name: Build the anydoc static archive
|
||||
run: ./scripts/build-anydoc-lib.sh
|
||||
|
||||
- name: Vet, test, and build with the engine linked in
|
||||
run: |
|
||||
go vet -tags anydoc ./internal/infrastructure/docparser/...
|
||||
go test -tags anydoc -count=1 ./internal/infrastructure/docparser/...
|
||||
go build -tags anydoc ./cmd/server
|
||||
|
|
|
|||
18
Makefile
18
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite
|
||||
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite anydoc-lib build-anydoc
|
||||
|
||||
# Show help
|
||||
help:
|
||||
|
|
@ -8,6 +8,8 @@ help:
|
|||
@echo " build 构建应用"
|
||||
@echo " run 运行应用"
|
||||
@echo " test 运行测试"
|
||||
@echo " anydoc-lib 构建 anydoc 静态库(需要 Rust 工具链)"
|
||||
@echo " build-anydoc 构建带 anydoc 解析引擎的应用"
|
||||
@echo " clean 清理构建文件"
|
||||
@echo ""
|
||||
@echo "Docker 命令:"
|
||||
|
|
@ -56,6 +58,7 @@ help:
|
|||
@echo " dev-logs 查看开发环境日志"
|
||||
@echo " dev-status 查看开发环境状态"
|
||||
@echo " dev-app 启动后端应用(本地运行,需先运行 dev-start)"
|
||||
@echo " 已 make anydoc-lib 时自动链接 anydoc 引擎"
|
||||
@echo " dev-frontend 启动前端(本地运行,需先运行 dev-start)"
|
||||
@echo ""
|
||||
@echo "Lite 模式(零外部依赖):"
|
||||
|
|
@ -87,6 +90,15 @@ endif
|
|||
build:
|
||||
go build -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
|
||||
# Build the anydoc static archive (Rust) that the `anydoc` build tag links.
|
||||
# Override the platform with TARGET=<rust-target-triple>.
|
||||
anydoc-lib:
|
||||
./scripts/build-anydoc-lib.sh
|
||||
|
||||
# Build the application with the in-process anydoc parser engine linked in.
|
||||
build-anydoc: anydoc-lib
|
||||
go build -tags anydoc -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
|
||||
# Run the application
|
||||
run: build
|
||||
./$(BINARY_NAME)
|
||||
|
|
@ -234,6 +246,8 @@ deps:
|
|||
|
||||
# Build for production
|
||||
# google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn for qdrant milvus proto conflict
|
||||
# GO_BUILD_TAGS adds optional build tags, e.g. GO_BUILD_TAGS=anydoc to link the
|
||||
# in-process office document parser (run `make anydoc-lib` first).
|
||||
build-prod:
|
||||
VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \
|
||||
COMMIT_ID=$${COMMIT_ID:-unknown}; \
|
||||
|
|
@ -243,7 +257,7 @@ build-prod:
|
|||
BUILD_TIME=$${BUILD_TIME:-unknown}; \
|
||||
GO_VERSION=$${GO_VERSION:-unknown}; \
|
||||
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
|
||||
go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
go build -tags "$(GO_BUILD_TAGS)" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
|
||||
# Build Lite version (single binary, SQLite + in-memory queue)
|
||||
# 会先构建前端到 web/,再构建 Go 二进制;SKIP_FRONTEND=1 可跳过前端
|
||||
|
|
|
|||
|
|
@ -43,8 +43,27 @@ ENV COMMIT_ID=${COMMIT_ID_ARG}
|
|||
ENV BUILD_TIME=${BUILD_TIME_ARG}
|
||||
ENV GO_VERSION=${GO_VERSION_ARG}
|
||||
|
||||
# Optional: link the anydoc parser engine, which converts office documents in
|
||||
# the Go process instead of the Python docreader. It is off by default because
|
||||
# it needs a Rust toolchain (a few minutes and ~1 GB of build-stage layers) for
|
||||
# an engine most deployments do not select.
|
||||
# docker build --build-arg WITH_ANYDOC=1 ...
|
||||
ARG WITH_ANYDOC=0
|
||||
ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo
|
||||
ENV PATH=/usr/local/cargo/bin:$PATH
|
||||
RUN if [ "$WITH_ANYDOC" = "1" ]; then \
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||
| sh -s -- -y --profile minimal --default-toolchain stable && \
|
||||
./scripts/build-anydoc-lib.sh; \
|
||||
fi
|
||||
|
||||
# Build the application with version info
|
||||
RUN --mount=type=cache,target=/go/pkg/mod make build-prod
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
if [ "$WITH_ANYDOC" = "1" ]; then \
|
||||
make build-prod GO_BUILD_TAGS=anydoc; \
|
||||
else \
|
||||
make build-prod; \
|
||||
fi
|
||||
RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/
|
||||
|
||||
# Final stage
|
||||
|
|
|
|||
|
|
@ -4976,6 +4976,10 @@ export default {
|
|||
name: 'Simple',
|
||||
desc: 'Simple format & image parsing (no external service required)'
|
||||
},
|
||||
anydoc: {
|
||||
name: 'anydoc',
|
||||
desc: 'In-process office document parser (no external service required)'
|
||||
},
|
||||
mineru: {
|
||||
name: 'MinerU',
|
||||
desc: 'MinerU self-hosted service'
|
||||
|
|
|
|||
|
|
@ -1399,6 +1399,10 @@ export default {
|
|||
name: 'Simple',
|
||||
desc: '간단한 형식 및 이미지 파싱 (외부 서비스 불필요)'
|
||||
},
|
||||
anydoc: {
|
||||
name: 'anydoc',
|
||||
desc: '프로세스 내 오피스 문서 파싱 (외부 서비스 불필요)'
|
||||
},
|
||||
builtin: {
|
||||
name: '내장',
|
||||
desc: 'DocReader 내장 파서 엔진 (docx/pdf/xlsx 등 복잡한 형식)'
|
||||
|
|
|
|||
|
|
@ -1399,6 +1399,10 @@ export default {
|
|||
name: 'Simple',
|
||||
desc: 'Простой формат и анализ изображений (внешний сервис не требуется)'
|
||||
},
|
||||
anydoc: {
|
||||
name: 'anydoc',
|
||||
desc: 'Разбор офисных документов внутри процесса (внешний сервис не требуется)'
|
||||
},
|
||||
builtin: {
|
||||
name: 'Встроенный',
|
||||
desc: 'Встроенный парсер DocReader (docx/pdf/xlsx и другие сложные форматы)'
|
||||
|
|
|
|||
|
|
@ -1401,6 +1401,10 @@ export default {
|
|||
name: 'Simple',
|
||||
desc: '简单格式 & 图片解析(无需外部服务)'
|
||||
},
|
||||
anydoc: {
|
||||
name: 'anydoc',
|
||||
desc: '进程内 Office 文档解析(无需外部服务)'
|
||||
},
|
||||
builtin: {
|
||||
name: '内置',
|
||||
desc: 'DocReader 内置解析引擎(docx/pdf/xlsx 等复杂格式)'
|
||||
|
|
|
|||
|
|
@ -478,11 +478,12 @@ const ENGINE_ORDER: Record<string, number> = {
|
|||
builtin: 0,
|
||||
weknoracloud: 1,
|
||||
simple: 2,
|
||||
markitdown: 3,
|
||||
mineru: 4,
|
||||
mineru_cloud: 5,
|
||||
paddleocr_vl: 6,
|
||||
paddleocr_vl_cloud: 7,
|
||||
anydoc: 3,
|
||||
markitdown: 4,
|
||||
mineru: 5,
|
||||
mineru_cloud: 6,
|
||||
paddleocr_vl: 7,
|
||||
paddleocr_vl_cloud: 8,
|
||||
}
|
||||
|
||||
const sortedEngines = computed(() => {
|
||||
|
|
|
|||
5
go.mod
5
go.mod
|
|
@ -152,6 +152,7 @@ require (
|
|||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/elastic/elastic-transport-go/v8 v8.9.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/firecrawl/anydoc/go v0.1.8
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
|
||||
|
|
@ -348,3 +349,7 @@ require (
|
|||
)
|
||||
|
||||
replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0
|
||||
|
||||
// The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still
|
||||
// open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z.
|
||||
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
|
||||
|
|
|
|||
|
|
@ -3752,41 +3752,23 @@ func isLikelyRateLimitError(err error) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// Returns nil when the required service is unavailable.
|
||||
func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader {
|
||||
switch engine {
|
||||
case docparser.SimpleEngineName:
|
||||
return &docparser.SimpleFormatReader{}
|
||||
case docparser.WeKnoraCloudEngineName:
|
||||
creds := s.tenantService.GetWeKnoraCloudCredentials(ctx)
|
||||
if creds == nil {
|
||||
logger.Warnf(ctx, "[resolveDocReader] WeKnoraCloud: no tenant credentials (fileType=%s)", fileType)
|
||||
return nil
|
||||
}
|
||||
reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(creds.AppID, creds.AppSecret)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[resolveDocReader] WeKnoraCloud reader init failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
return reader
|
||||
case "mineru":
|
||||
return docparser.NewMinerUReader(overrides)
|
||||
case "mineru_cloud":
|
||||
return docparser.NewMinerUCloudReader(overrides)
|
||||
case "paddleocr_vl":
|
||||
return docparser.NewPaddleOCRVLReader(overrides)
|
||||
case "paddleocr_vl_cloud":
|
||||
return docparser.NewPaddleOCRVLCloudReader(overrides)
|
||||
case "builtin":
|
||||
// 明确指定使用 builtin 引擎(docreader),不使用 simple format 兜底
|
||||
return s.documentReader
|
||||
default:
|
||||
// 未指定引擎时的兜底逻辑:simple format 使用 Go 原生处理,其他使用 docreader
|
||||
if !isURL && docparser.IsSimpleFormat(fileType) {
|
||||
return &docparser.SimpleFormatReader{}
|
||||
}
|
||||
return s.documentReader
|
||||
// resolveDocReader picks the reader for one parse request. The engine catalog
|
||||
// itself lives in the docparser registry; this only supplies the dependencies
|
||||
// the service owns. Returns nil when the chosen engine cannot run — an
|
||||
// unconfigured cloud engine, a disconnected docreader — after logging why.
|
||||
func (s *knowledgeService) resolveDocReader(
|
||||
ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string,
|
||||
) interfaces.DocReader {
|
||||
reader, err := docparser.NewReader(ctx, engine, fileType, isURL, docparser.ReaderDeps{
|
||||
Overrides: overrides,
|
||||
Remote: s.documentReader,
|
||||
WeKnoraCloudCredentials: s.tenantService.GetWeKnoraCloudCredentials,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[resolveDocReader] engine=%q fileType=%q unusable: %v", engine, fileType, err)
|
||||
return nil
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
// failKnowledge marks knowledge as failed (only on last retry) and returns an error.
|
||||
|
|
|
|||
|
|
@ -394,14 +394,15 @@ func (s *temporaryDocumentService) parse(ctx context.Context, document *types.Te
|
|||
if tenant, ok := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil {
|
||||
request.ParserEngineOverrides = tenant.ParserEngineConfig.ToOverridesMap()
|
||||
}
|
||||
var result *types.ReadResult
|
||||
if docparser.IsSimpleFormat(ext) && (request.ParserEngine == "" || request.ParserEngine == "auto") {
|
||||
result, err = (&docparser.SimpleFormatReader{}).Read(ctx, request)
|
||||
} else if s.documentReader != nil {
|
||||
result, err = s.documentReader.Read(ctx, request)
|
||||
} else {
|
||||
err = fmt.Errorf("document reader is not configured")
|
||||
deps := docparser.ReaderDeps{Overrides: request.ParserEngineOverrides, Remote: s.documentReader}
|
||||
if s.tenantService != nil {
|
||||
deps.WeKnoraCloudCredentials = s.tenantService.GetWeKnoraCloudCredentials
|
||||
}
|
||||
reader, err := docparser.NewReader(ctx, parserEngine, strings.TrimPrefix(ext, "."), false, deps)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("parse document: %w", err)
|
||||
}
|
||||
result, err := reader.Read(ctx, request)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("parse document: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -244,16 +244,31 @@ func (p *AttachmentProcessor) processWithDocumentReader(
|
|||
parserEngine = s
|
||||
}
|
||||
}
|
||||
overrides := getParserEngineOverridesFromContext(ctx)
|
||||
|
||||
result, err := p.documentReader.Read(ctx, &types.ReadRequest{
|
||||
// Engines that parse in this process (anydoc, MinerU, ...) are resolved
|
||||
// through the registry so a chat attachment honours the same engine rules
|
||||
// as an ingested document. Anything the registry cannot build here — a
|
||||
// cloud engine whose credentials this path cannot resolve — falls back to
|
||||
// the docreader, which is where every engine name went before.
|
||||
reader, err := docparser.NewReader(ctx, parserEngine, normalizedType, false, docparser.ReaderDeps{
|
||||
Overrides: overrides,
|
||||
Remote: p.documentReader,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "parser engine %q unusable for this attachment, using docreader: %v", parserEngine, err)
|
||||
reader = p.documentReader
|
||||
}
|
||||
|
||||
result, err := reader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: data,
|
||||
FileName: fileName,
|
||||
FileType: normalizedType,
|
||||
ParserEngine: parserEngine,
|
||||
ParserEngineOverrides: getParserEngineOverridesFromContext(ctx),
|
||||
ParserEngineOverrides: overrides,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("DocumentReader failed: %w", err)
|
||||
return fmt.Errorf("document parsing failed: %w", err)
|
||||
}
|
||||
|
||||
// Resolve embedded image refs to storage URLs.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
// Package anydoc converts office documents to Markdown inside the Go process,
|
||||
// without the Python docreader service.
|
||||
//
|
||||
// The conversion itself is the anydoc Rust library, linked as a static archive
|
||||
// through cgo (see third_party/anydoc-go). That archive needs a Rust toolchain
|
||||
// to produce, so it is opt-in: only builds tagged `anydoc` link it. Every other
|
||||
// build compiles the stub in backend_stub.go, where Available reports false and
|
||||
// the engine registry hides the engine with a reason the UI can show.
|
||||
//
|
||||
// The exported surface is deliberately narrow — bytes in, Markdown and embedded
|
||||
// images out — so the backend can be replaced (a WASI runtime, a future
|
||||
// upstream Go module) without touching the parser code that calls it.
|
||||
package anydoc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ImageDir is the markdown path prefix for extracted images. The image
|
||||
// resolver matches references by this path and swaps them for storage URLs.
|
||||
const ImageDir = "images/"
|
||||
|
||||
// Result is one converted document.
|
||||
type Result struct {
|
||||
// Markdown is GitHub-Flavored Markdown for the whole document.
|
||||
// When WithAssets is set, embedded images appear in place as
|
||||
// `` via anydoc's official serializer.
|
||||
Markdown string
|
||||
// Assets are the images embedded in the document, in document order.
|
||||
// Always empty for PDF, which anydoc renders straight to Markdown
|
||||
// without a document model.
|
||||
Assets []Asset
|
||||
// AssetsError explains why Assets is empty when images were asked for.
|
||||
// The document-model parse is what both extracts images and places them
|
||||
// in the markdown; when it fails, Markdown still comes from the text-only
|
||||
// renderer so the conversion succeeds and the caller decides whether to
|
||||
// log or ignore the loss.
|
||||
AssetsError error
|
||||
}
|
||||
|
||||
// Asset is one image embedded in a document.
|
||||
type Asset struct {
|
||||
// ID is the document-model asset index, as referenced by in-place image
|
||||
// sources. It is stable for a single conversion.
|
||||
ID uint64
|
||||
// Name is a generated, extension-carrying file name ("image-1.png"):
|
||||
// embedded assets have no name of their own inside the container.
|
||||
Name string
|
||||
// MediaType is the IANA media type the container declared.
|
||||
MediaType string
|
||||
// Data is the raw image bytes.
|
||||
Data []byte
|
||||
// Alt is the image's alternative text as the document author wrote it,
|
||||
// empty when the document carries none.
|
||||
Alt string
|
||||
// Section is the text of the nearest heading above the image, empty when
|
||||
// the image sits before the first heading.
|
||||
Section string
|
||||
}
|
||||
|
||||
// Options tunes a single conversion.
|
||||
type Options struct {
|
||||
// Format names the parser explicitly ("docx", "csv", ...). Empty means
|
||||
// the format is detected from the content, which every format except
|
||||
// CSV carries a signature for.
|
||||
Format string
|
||||
// WithAssets extracts embedded images and renders them in place in the
|
||||
// markdown using anydoc's official GFM serializer (asset images are
|
||||
// rewritten to `images/image-N.ext` links first). Ignored for PDF.
|
||||
WithAssets bool
|
||||
}
|
||||
|
||||
// ErrUnavailable is returned by Convert when the binding is not linked into
|
||||
// this build. Callers that can fall back to another engine should check for it
|
||||
// with errors.Is.
|
||||
var ErrUnavailable = errors.New("anydoc: binding not built into this binary")
|
||||
|
||||
// supportedExtensions are the file types anydoc converts, mapped to the format
|
||||
// name its parser selector uses. The list is static so that engine metadata
|
||||
// (file types shown in the UI) is identical whether or not the binding is
|
||||
// linked in.
|
||||
var supportedExtensions = map[string]string{
|
||||
"doc": "doc",
|
||||
"docx": "docx",
|
||||
"docm": "docx",
|
||||
"odt": "odt",
|
||||
"rtf": "rtf",
|
||||
"ppt": "ppt",
|
||||
"pptx": "pptx",
|
||||
"pptm": "pptx",
|
||||
"odp": "odp",
|
||||
"xls": "xlsx",
|
||||
"xlsx": "xlsx",
|
||||
"xlsm": "xlsx",
|
||||
"ods": "ods",
|
||||
"epub": "epub",
|
||||
"csv": "csv",
|
||||
"pdf": "pdf",
|
||||
}
|
||||
|
||||
// SupportedFileTypes returns the extensions anydoc can convert, sorted so the
|
||||
// engine list is stable across restarts.
|
||||
func SupportedFileTypes() []string {
|
||||
types := make([]string, 0, len(supportedExtensions))
|
||||
for ext := range supportedExtensions {
|
||||
types = append(types, ext)
|
||||
}
|
||||
slices.Sort(types)
|
||||
return types
|
||||
}
|
||||
|
||||
// FormatForFile resolves the anydoc format name for a file type or file name.
|
||||
// ok is false for anything anydoc does not convert.
|
||||
func FormatForFile(fileType, fileName string) (format string, ok bool) {
|
||||
ext := normalizeExt(fileType)
|
||||
if ext == "" {
|
||||
ext = normalizeExt(filepath.Ext(fileName))
|
||||
}
|
||||
format, ok = supportedExtensions[ext]
|
||||
return format, ok
|
||||
}
|
||||
|
||||
// Supports reports whether anydoc converts this file type, regardless of
|
||||
// whether the binding is linked into this build.
|
||||
func Supports(fileType, fileName string) bool {
|
||||
_, ok := FormatForFile(fileType, fileName)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Available reports whether conversions can actually run in this binary.
|
||||
func Available() bool { return backendAvailable() }
|
||||
|
||||
// UnavailableReason explains, for the UI, why Available is false. It returns
|
||||
// "" when the binding is available.
|
||||
func UnavailableReason() string { return backendUnavailableReason() }
|
||||
|
||||
// Version reports the anydoc release the linked binding was built from, or ""
|
||||
// when no binding is linked.
|
||||
func Version() string { return backendVersion() }
|
||||
|
||||
// Convert turns document bytes into Markdown. It returns ErrUnavailable when
|
||||
// the binding is not linked into this build.
|
||||
func Convert(data []byte, opts Options) (*Result, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("anydoc: empty document")
|
||||
}
|
||||
if !backendAvailable() {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnavailable, backendUnavailableReason())
|
||||
}
|
||||
// PDF has no document model, so asset extraction would only produce an
|
||||
// error the caller has to special-case. Drop the request instead.
|
||||
if opts.Format == "pdf" {
|
||||
opts.WithAssets = false
|
||||
}
|
||||
return backendConvert(data, opts)
|
||||
}
|
||||
|
||||
// PDFNeedsOCR reports whether a Convert error means the PDF has no usable
|
||||
// text layer. Scanned exam papers fail this way; the caller should fall back
|
||||
// to an engine that can rasterize pages for OCR.
|
||||
func PDFNeedsOCR(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "ocr is required") ||
|
||||
strings.Contains(msg, "no extractable text")
|
||||
}
|
||||
|
||||
func normalizeExt(s string) string {
|
||||
return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(s), "."))
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
package anydoc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFormatForFileResolvesTypeAndName(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
fileType string
|
||||
fileName string
|
||||
wantFormat string
|
||||
wantOK bool
|
||||
}{
|
||||
{name: "file type", fileType: "docx", wantFormat: "docx", wantOK: true},
|
||||
{name: "dotted file type", fileType: ".DOCX", wantFormat: "docx", wantOK: true},
|
||||
{name: "macro format maps to base format", fileType: "pptm", wantFormat: "pptx", wantOK: true},
|
||||
{name: "legacy excel maps to xlsx parser", fileType: "xls", wantFormat: "xlsx", wantOK: true},
|
||||
{name: "falls back to file name", fileName: "report.odt", wantFormat: "odt", wantOK: true},
|
||||
{name: "unsupported type", fileType: "png", wantOK: false},
|
||||
{name: "nothing to go on", wantOK: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
format, ok := FormatForFile(tc.fileType, tc.fileName)
|
||||
if ok != tc.wantOK {
|
||||
t.Fatalf("FormatForFile(%q, %q) ok = %v, want %v", tc.fileType, tc.fileName, ok, tc.wantOK)
|
||||
}
|
||||
if format != tc.wantFormat {
|
||||
t.Fatalf("FormatForFile(%q, %q) = %q, want %q", tc.fileType, tc.fileName, format, tc.wantFormat)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The engine list must look the same whether or not the converter is linked
|
||||
// in, so that the UI can show the file types with a "not built in" reason.
|
||||
func TestSupportedFileTypesIsStableAndSorted(t *testing.T) {
|
||||
types := SupportedFileTypes()
|
||||
if !slices.IsSorted(types) {
|
||||
t.Fatalf("SupportedFileTypes is not sorted: %v", types)
|
||||
}
|
||||
for _, want := range []string{"docx", "pptx", "xlsx", "epub", "pdf", "csv"} {
|
||||
if !slices.Contains(types, want) {
|
||||
t.Errorf("SupportedFileTypes does not include %q: %v", want, types)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPDFNeedsOCR(t *testing.T) {
|
||||
if PDFNeedsOCR(nil) {
|
||||
t.Fatal("nil error was treated as needing OCR")
|
||||
}
|
||||
if !PDFNeedsOCR(errors.New("PDF has no extractable text (Scanned, 5 pages): OCR is required")) {
|
||||
t.Error("scanned-PDF error was not detected")
|
||||
}
|
||||
if PDFNeedsOCR(errors.New("not a readable zip archive")) {
|
||||
t.Error("unrelated error was treated as needing OCR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertRejectsEmptyInput(t *testing.T) {
|
||||
if _, err := Convert(nil, Options{Format: "docx"}); err == nil {
|
||||
t.Fatal("Convert(nil) succeeded, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// Without the build tag there is no converter, and callers must be able to
|
||||
// detect that so they can fall back to another engine.
|
||||
func TestConvertReportsUnavailableBackend(t *testing.T) {
|
||||
if Available() {
|
||||
t.Skip("converter is linked into this build")
|
||||
}
|
||||
_, err := Convert([]byte("id,name\n1,a\n"), Options{Format: "csv"})
|
||||
if !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("Convert error = %v, want ErrUnavailable", err)
|
||||
}
|
||||
if UnavailableReason() == "" {
|
||||
t.Error("UnavailableReason is empty while the converter is unavailable")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
//go:build anydoc && cgo
|
||||
|
||||
package anydoc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mime"
|
||||
"strings"
|
||||
|
||||
upstream "github.com/firecrawl/anydoc/go"
|
||||
)
|
||||
|
||||
// This build links the anydoc Rust archive through the upstream cgo bindings
|
||||
// (vendored under third_party/anydoc-go until they are published as a module).
|
||||
// Conversion runs in-process: no subprocess, no HTTP hop, no Python.
|
||||
|
||||
func backendAvailable() bool { return true }
|
||||
|
||||
func backendUnavailableReason() string { return "" }
|
||||
|
||||
func backendVersion() string { return upstream.Version }
|
||||
|
||||
func backendConvert(data []byte, opts Options) (*Result, error) {
|
||||
format, err := upstreamFormat(opts.Format)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !opts.WithAssets {
|
||||
markdown, err := upstream.ToMarkdownBytes(data, format)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err)
|
||||
}
|
||||
return &Result{Markdown: markdown}, nil
|
||||
}
|
||||
|
||||
// Official GFM serializer after rewriting Asset images to External URLs
|
||||
// (`images/image-N.ext`). ToDocument still supplies the image bytes.
|
||||
document, err := upstream.ToDocument(data, format)
|
||||
if err != nil {
|
||||
markdown, mdErr := upstream.ToMarkdownBytes(data, format)
|
||||
if mdErr != nil {
|
||||
return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", mdErr)
|
||||
}
|
||||
return &Result{
|
||||
Markdown: markdown,
|
||||
AssetsError: fmt.Errorf("anydoc: image extraction failed: %w", err),
|
||||
}, nil
|
||||
}
|
||||
markdown, err := upstream.ToMarkdownWithAssetLinks(data, format)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err)
|
||||
}
|
||||
return &Result{Markdown: markdown, Assets: collectAssets(document)}, nil
|
||||
}
|
||||
|
||||
// upstreamFormat maps our format name onto the binding's constant. An empty
|
||||
// name means "detect from content".
|
||||
func upstreamFormat(name string) (*upstream.Format, error) {
|
||||
if name == "" {
|
||||
return nil, nil
|
||||
}
|
||||
format, ok := upstream.FormatFromExtension(name)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("anydoc: unsupported format %q", name)
|
||||
}
|
||||
return &format, nil
|
||||
}
|
||||
|
||||
// collectAssets names each embedded image after its position in the document,
|
||||
// because containers store assets without a usable file name of their own, and
|
||||
// attaches the alt text and section the image appeared under.
|
||||
func collectAssets(document *upstream.Document) []Asset {
|
||||
if document == nil || len(document.Assets) == 0 {
|
||||
return nil
|
||||
}
|
||||
placements := placeAssets(document.Blocks)
|
||||
assets := make([]Asset, 0, len(document.Assets))
|
||||
for i, asset := range document.Assets {
|
||||
if len(asset.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
placement := placements[uint64(i)]
|
||||
assets = append(assets, Asset{
|
||||
ID: uint64(i),
|
||||
Name: fmt.Sprintf("image-%d%s", i+1, extensionFor(asset.MediaType)),
|
||||
MediaType: asset.MediaType,
|
||||
Data: asset.Data,
|
||||
Alt: placement.alt,
|
||||
Section: placement.section,
|
||||
})
|
||||
}
|
||||
return assets
|
||||
}
|
||||
|
||||
// placement is where in the document an asset was referenced from.
|
||||
type placement struct {
|
||||
alt string
|
||||
section string
|
||||
}
|
||||
|
||||
// placeAssets walks the document body and records, per asset, the alt text of
|
||||
// the image that referenced it and the heading it sat under. The walk
|
||||
// recurses into lists, tables, and quotes, since an image can be nested
|
||||
// anywhere a paragraph can.
|
||||
func placeAssets(blocks []upstream.Block) map[uint64]placement {
|
||||
placements := make(map[uint64]placement)
|
||||
section := ""
|
||||
walkBlocks(blocks, §ion, placements)
|
||||
return placements
|
||||
}
|
||||
|
||||
func walkBlocks(blocks []upstream.Block, section *string, placements map[uint64]placement) {
|
||||
for _, block := range blocks {
|
||||
switch block.Kind {
|
||||
case "heading":
|
||||
*section = inlineText(block.Content)
|
||||
walkInlines(block.Content, *section, placements)
|
||||
case "paragraph":
|
||||
walkInlines(block.Content, *section, placements)
|
||||
case "block_quote":
|
||||
walkBlocks(block.Blocks, section, placements)
|
||||
case "list":
|
||||
if block.List != nil {
|
||||
for _, item := range block.List.Items {
|
||||
walkBlocks(item.Blocks, section, placements)
|
||||
}
|
||||
}
|
||||
case "table":
|
||||
if block.Table != nil {
|
||||
for _, row := range block.Table.Grid {
|
||||
for _, slot := range row {
|
||||
if slot.Cell != nil {
|
||||
walkBlocks(slot.Cell.Blocks, section, placements)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func walkInlines(inlines []upstream.Inline, section string, placements map[uint64]placement) {
|
||||
for _, inline := range inlines {
|
||||
switch inline.Kind {
|
||||
case "image":
|
||||
if inline.Source == nil || inline.Source.Kind != "asset" || inline.Source.AssetID == nil {
|
||||
continue
|
||||
}
|
||||
alt := ""
|
||||
if inline.Alt != nil {
|
||||
alt = strings.TrimSpace(*inline.Alt)
|
||||
}
|
||||
// First reference wins: the same asset can be placed more than
|
||||
// once, and the earliest placement is the one a reader would
|
||||
// associate it with.
|
||||
if _, seen := placements[*inline.Source.AssetID]; !seen {
|
||||
placements[*inline.Source.AssetID] = placement{alt: alt, section: section}
|
||||
}
|
||||
case "link":
|
||||
walkInlines(inline.Content, section, placements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inlineText flattens inline content to plain text, which is all a heading is
|
||||
// needed for here.
|
||||
func inlineText(inlines []upstream.Inline) string {
|
||||
var text strings.Builder
|
||||
for _, inline := range inlines {
|
||||
switch inline.Kind {
|
||||
case "text":
|
||||
if inline.Text != nil {
|
||||
text.WriteString(*inline.Text)
|
||||
}
|
||||
case "link":
|
||||
text.WriteString(inlineText(inline.Content))
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(text.String())
|
||||
}
|
||||
|
||||
func extensionFor(mediaType string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(mediaType)) {
|
||||
case "image/jpeg", "image/jpg":
|
||||
return ".jpg"
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
case "image/bmp":
|
||||
return ".bmp"
|
||||
case "image/tiff":
|
||||
return ".tiff"
|
||||
case "image/svg+xml":
|
||||
return ".svg"
|
||||
}
|
||||
// Anything else (EMF/WMF drawings, unknown object payloads) keeps
|
||||
// whatever extension the media type registry knows, and .bin otherwise,
|
||||
// so downstream storage never writes an extension-less blob.
|
||||
if exts, err := mime.ExtensionsByType(mediaType); err == nil && len(exts) > 0 {
|
||||
return exts[0]
|
||||
}
|
||||
return ".bin"
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
//go:build !anydoc || !cgo
|
||||
|
||||
package anydoc
|
||||
|
||||
// The default build links no converter: the anydoc archive is a Rust build
|
||||
// artifact, and requiring a Rust toolchain to compile WeKnora would be a steep
|
||||
// price for one optional engine. Everything still compiles and the engine
|
||||
// registry simply reports the engine as unavailable.
|
||||
|
||||
const unavailableReason = "anydoc engine not built into this binary " +
|
||||
"(rebuild with `make build-anydoc` / `-tags anydoc`)"
|
||||
|
||||
func backendAvailable() bool { return false }
|
||||
|
||||
func backendUnavailableReason() string { return unavailableReason }
|
||||
|
||||
func backendVersion() string { return "" }
|
||||
|
||||
func backendConvert(_ []byte, _ Options) (*Result, error) {
|
||||
// Convert checks Available first, so this is only reachable if a caller
|
||||
// bypasses it.
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
|
|
@ -0,0 +1,333 @@
|
|||
//go:build anydoc && cgo
|
||||
|
||||
package anydoc
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// These tests exercise the linked Rust converter, so they only build with the
|
||||
// `anydoc` tag. Run them with:
|
||||
//
|
||||
// scripts/build-anydoc-lib.sh && go test -tags anydoc ./internal/infrastructure/docparser/...
|
||||
|
||||
// SupportedFileTypes is a static list, so that the engine can advertise its
|
||||
// file types in builds that link no converter. This is the check that it still
|
||||
// agrees with what the converter actually accepts, in either direction.
|
||||
func TestSupportedFileTypesMatchTheConverter(t *testing.T) {
|
||||
for _, fileType := range SupportedFileTypes() {
|
||||
format, ok := FormatForFile(fileType, "")
|
||||
if !ok {
|
||||
t.Errorf("%q is advertised but maps to no format", fileType)
|
||||
continue
|
||||
}
|
||||
if _, err := upstreamFormat(format); err != nil {
|
||||
t.Errorf("%q maps to format %q, which the converter rejects: %v", fileType, format, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertCSV(t *testing.T) {
|
||||
result, err := Convert([]byte("quarter,widgets\nQ1,12\nQ2,15\n"), Options{Format: "csv"})
|
||||
if err != nil {
|
||||
t.Fatalf("Convert: %v", err)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "| quarter | widgets |") {
|
||||
t.Fatalf("expected a markdown table header, got:\n%s", result.Markdown)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "| Q1 | 12 |") {
|
||||
t.Fatalf("expected the first data row, got:\n%s", result.Markdown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertDocxWithEmbeddedImage(t *testing.T) {
|
||||
document := buildDocx(t)
|
||||
|
||||
result, err := Convert(document, Options{Format: "docx", WithAssets: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Convert: %v", err)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "# Quarterly report") {
|
||||
t.Fatalf("expected the heading, got:\n%s", result.Markdown)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "Widgets shipped on time.") {
|
||||
t.Fatalf("expected the body paragraph, got:\n%s", result.Markdown)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "Closing remarks.") {
|
||||
t.Fatalf("expected the trailing paragraph, got:\n%s", result.Markdown)
|
||||
}
|
||||
if len(result.Assets) != 1 {
|
||||
t.Fatalf("got %d assets, want 1", len(result.Assets))
|
||||
}
|
||||
asset := result.Assets[0]
|
||||
if asset.Name != "image-1.png" {
|
||||
t.Errorf("asset name = %q, want image-1.png", asset.Name)
|
||||
}
|
||||
if !bytes.Equal(asset.Data, onePixelPNG()) {
|
||||
t.Errorf("asset data does not round-trip the embedded image")
|
||||
}
|
||||
if asset.Alt != "Shipping chart" {
|
||||
t.Errorf("asset alt = %q, want %q", asset.Alt, "Shipping chart")
|
||||
}
|
||||
if asset.Section != "Quarterly report" {
|
||||
t.Errorf("asset section = %q, want %q", asset.Section, "Quarterly report")
|
||||
}
|
||||
imageLink := ""
|
||||
if !strings.Contains(result.Markdown, imageLink) {
|
||||
t.Fatalf("expected the image in place, got:\n%s", result.Markdown)
|
||||
}
|
||||
before := strings.Index(result.Markdown, "Widgets shipped on time.")
|
||||
at := strings.Index(result.Markdown, imageLink)
|
||||
after := strings.Index(result.Markdown, "Closing remarks.")
|
||||
if before < 0 || at < 0 || after < 0 || !(before < at && at < after) {
|
||||
t.Fatalf("image is not between the surrounding paragraphs:\n%s", result.Markdown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertTextlessPDFNeedsOCR(t *testing.T) {
|
||||
_, err := Convert(textlessPDF(), Options{Format: "pdf"})
|
||||
if err == nil {
|
||||
t.Fatal("Convert succeeded on a PDF with no text, want an error")
|
||||
}
|
||||
if !PDFNeedsOCR(err) {
|
||||
t.Fatalf("Convert error = %v, want an OCR-required error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The ABI reports error messages through a thread-local slot that a second
|
||||
// call reads, so a goroutine resumed on another OS thread would report an
|
||||
// empty or someone else's message. The binding pins the goroutine for the
|
||||
// pair; this is the regression test for that.
|
||||
func TestErrorDetailSurvivesConcurrency(t *testing.T) {
|
||||
const conversions = 4000
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var empty, crossed atomic.Int64
|
||||
for i := 0; i < conversions; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
// docx and xlsx failures carry distinctive messages, so a detail
|
||||
// naming the other format proves it came from the wrong thread.
|
||||
format, want, other := "docx", "not a readable zip archive", "unreadable workbook"
|
||||
if i%2 == 1 {
|
||||
format, want, other = "xlsx", "unreadable workbook", "not a readable zip archive"
|
||||
}
|
||||
_, err := Convert([]byte(fmt.Sprintf("garbage %d", i)), Options{Format: format})
|
||||
if err == nil {
|
||||
t.Errorf("Convert(garbage) succeeded")
|
||||
return
|
||||
}
|
||||
detail := err.Error()
|
||||
switch {
|
||||
case strings.Contains(detail, other):
|
||||
crossed.Add(1)
|
||||
case !strings.Contains(detail, want):
|
||||
empty.Add(1)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if empty.Load() > 0 || crossed.Load() > 0 {
|
||||
t.Fatalf("error details lost or crossed between goroutines: lost=%d crossed=%d (of %d)",
|
||||
empty.Load(), crossed.Load(), conversions)
|
||||
}
|
||||
}
|
||||
|
||||
// Detection reads the container itself, so a document whose format is not
|
||||
// named still converts.
|
||||
func TestConvertDetectsFormatFromContent(t *testing.T) {
|
||||
result, err := Convert(buildDocx(t), Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Convert: %v", err)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "# Quarterly report") {
|
||||
t.Fatalf("expected the heading, got:\n%s", result.Markdown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertRejectsGarbage(t *testing.T) {
|
||||
if _, err := Convert([]byte("not a document at all"), Options{Format: "docx"}); err == nil {
|
||||
t.Fatal("Convert succeeded on garbage input, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// PDF has no document model, so WithAssets has to be dropped rather than turn
|
||||
// a working conversion into an error.
|
||||
func TestConvertPDFIgnoresAssetRequest(t *testing.T) {
|
||||
result, err := Convert(minimalPDF(), Options{Format: "pdf", WithAssets: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Convert: %v", err)
|
||||
}
|
||||
if !strings.Contains(result.Markdown, "Shipping summary") {
|
||||
t.Fatalf("expected the page text, got:\n%s", result.Markdown)
|
||||
}
|
||||
if len(result.Assets) != 0 {
|
||||
t.Errorf("got %d assets for a PDF, want 0", len(result.Assets))
|
||||
}
|
||||
}
|
||||
|
||||
// A PDF whose catalog holds a deeply nested array is the shape of
|
||||
// RUSTSEC-2026-0187: lopdf 0.41 recursed per nesting level and killed the
|
||||
// process with an uncatchable stack overflow, which no Go-side or Rust-side
|
||||
// recover can contain. The pinned dependency in third_party/anydoc-go bounds
|
||||
// the recursion, so this must come back as an ordinary error — and if a
|
||||
// dependency bump ever reintroduces it, this test crashes rather than passing
|
||||
// quietly.
|
||||
func TestDeeplyNestedPDFFailsWithoutKillingTheProcess(t *testing.T) {
|
||||
const depth = 50000
|
||||
nested := strings.Repeat("[", depth) + strings.Repeat("]", depth)
|
||||
|
||||
_, err := Convert(nestedPDF(nested), Options{Format: "pdf"})
|
||||
if err == nil {
|
||||
t.Fatal("Convert succeeded on a PDF with no text, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// buildDocx writes the smallest OOXML package that carries a heading, a
|
||||
// paragraph, and one embedded image.
|
||||
func buildDocx(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
parts := map[string]string{
|
||||
"[Content_Types].xml": `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Default Extension="png" ContentType="image/png"/>
|
||||
<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
|
||||
<Override PartName="/word/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"/>
|
||||
</Types>`,
|
||||
"_rels/.rels": `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
|
||||
</Relationships>`,
|
||||
"word/styles.xml": `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
|
||||
<w:style w:type="paragraph" w:styleId="Heading1">
|
||||
<w:name w:val="heading 1"/>
|
||||
<w:pPr><w:outlineLvl w:val="0"/></w:pPr>
|
||||
</w:style>
|
||||
</w:styles>`,
|
||||
"word/_rels/document.xml.rels": `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>
|
||||
<Relationship Id="rId10" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/image1.png"/>
|
||||
</Relationships>`,
|
||||
"word/document.xml": `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
||||
xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"
|
||||
xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"
|
||||
xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture">
|
||||
<w:body>
|
||||
<w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>Quarterly report</w:t></w:r></w:p>
|
||||
<w:p><w:r><w:t>Widgets shipped on time.</w:t></w:r></w:p>
|
||||
<w:p><w:r><w:drawing><wp:inline><wp:docPr id="1" name="Chart" descr="Shipping chart"/>
|
||||
<a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/picture">
|
||||
<pic:pic><pic:nvPicPr><pic:cNvPr id="1" name="Chart"/><pic:cNvPicPr/></pic:nvPicPr>
|
||||
<pic:blipFill><a:blip r:embed="rId10"/></pic:blipFill>
|
||||
<pic:spPr/></pic:pic>
|
||||
</a:graphicData></a:graphic>
|
||||
</wp:inline></w:drawing></w:r></w:p>
|
||||
<w:p><w:r><w:t>Closing remarks.</w:t></w:r></w:p>
|
||||
</w:body>
|
||||
</w:document>`,
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
archive := zip.NewWriter(&buf)
|
||||
for name, content := range parts {
|
||||
writer, err := archive.Create(name)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", name, err)
|
||||
}
|
||||
if _, err := writer.Write([]byte(content)); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
image, err := archive.Create("word/media/image1.png")
|
||||
if err != nil {
|
||||
t.Fatalf("create image part: %v", err)
|
||||
}
|
||||
if _, err := image.Write(onePixelPNG()); err != nil {
|
||||
t.Fatalf("write image part: %v", err)
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
t.Fatalf("close archive: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// onePixelPNG is a 1x1 transparent PNG: the smallest thing a container will
|
||||
// accept as an image part.
|
||||
func onePixelPNG() []byte {
|
||||
return []byte{
|
||||
0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a,
|
||||
0x00, 0x00, 0x00, 0x0d, 'I', 'H', 'D', 'R',
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
||||
0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
|
||||
0x00, 0x00, 0x00, 0x0a, 'I', 'D', 'A', 'T',
|
||||
0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, 0x05, 0x00, 0x01,
|
||||
0x0d, 0x0a, 0x2d, 0xb4,
|
||||
0x00, 0x00, 0x00, 0x00, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82,
|
||||
}
|
||||
}
|
||||
|
||||
// minimalPDF is a one-page PDF with a single text run, written by hand so the
|
||||
// test carries no binary fixture.
|
||||
func minimalPDF() []byte {
|
||||
content := "BT /F1 12 Tf 20 100 Td (Shipping summary) Tj ET\n"
|
||||
return writePDF([]string{
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R " +
|
||||
"/Resources << /Font << /F1 5 0 R >> >> >>",
|
||||
fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(content), content),
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
})
|
||||
}
|
||||
|
||||
// nestedPDF is a one-page PDF carrying the given nested value in its catalog.
|
||||
func nestedPDF(nested string) []byte {
|
||||
return writePDF([]string{
|
||||
"<< /Type /Catalog /Pages 2 0 R /Nested " + nested + " >>",
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>",
|
||||
})
|
||||
}
|
||||
|
||||
// textlessPDF is a one-page PDF with no text stream: the shape of a scanned
|
||||
// exam paper as far as pdf-inspector is concerned.
|
||||
func textlessPDF() []byte {
|
||||
return writePDF([]string{
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>",
|
||||
})
|
||||
}
|
||||
|
||||
func writePDF(objects []string) []byte {
|
||||
var pdf bytes.Buffer
|
||||
offsets := make([]int, len(objects))
|
||||
pdf.WriteString("%PDF-1.4\n")
|
||||
for i, object := range objects {
|
||||
offsets[i] = pdf.Len()
|
||||
fmt.Fprintf(&pdf, "%d 0 obj\n%s\nendobj\n", i+1, object)
|
||||
}
|
||||
|
||||
xref := pdf.Len()
|
||||
fmt.Fprintf(&pdf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1)
|
||||
for _, offset := range offsets {
|
||||
fmt.Fprintf(&pdf, "%010d 00000 n \n", offset)
|
||||
}
|
||||
fmt.Fprintf(&pdf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref)
|
||||
return pdf.Bytes()
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// AnydocReader converts office documents to markdown in this process, using
|
||||
// the anydoc converter instead of the Python docreader service.
|
||||
//
|
||||
// Embedded images are rendered in place from the document model. PDFs with no
|
||||
// text layer (scanned exam papers) cannot be converted by anydoc; when a
|
||||
// fallback reader is configured (typically the builtin docreader) those files
|
||||
// are handed off so pages can be rasterized for OCR.
|
||||
type AnydocReader struct {
|
||||
// extractImages controls whether embedded images are parsed and placed
|
||||
// in the markdown. Off halves parse time for text-only indexing.
|
||||
extractImages bool
|
||||
// fallback is used when a PDF has no extractable text. Nil means the
|
||||
// conversion error is returned to the caller.
|
||||
fallback interfaces.DocReader
|
||||
}
|
||||
|
||||
// NewAnydocReader builds a reader. The "anydoc_extract_images" override turns
|
||||
// off image extraction. fallback may be nil.
|
||||
func NewAnydocReader(overrides map[string]string, fallback interfaces.DocReader) *AnydocReader {
|
||||
return &AnydocReader{
|
||||
extractImages: !isFalsey(overrides["anydoc_extract_images"]),
|
||||
fallback: fallback,
|
||||
}
|
||||
}
|
||||
|
||||
// Read converts the document carried by the request.
|
||||
func (r *AnydocReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
if req.URL != "" && len(req.FileContent) == 0 {
|
||||
return nil, fmt.Errorf("anydoc engine reads uploaded documents, not URLs")
|
||||
}
|
||||
|
||||
format, ok := anydoc.FormatForFile(req.FileType, req.FileName)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("anydoc engine does not support file type %q", fileTypeOf(req))
|
||||
}
|
||||
|
||||
converted, err := anydoc.Convert(req.FileContent, anydoc.Options{
|
||||
Format: format,
|
||||
WithAssets: r.extractImages,
|
||||
})
|
||||
if format == "pdf" && (anydoc.PDFNeedsOCR(err) || pdfMarkdownIsEmpty(converted, err)) {
|
||||
return r.readScannedPDF(ctx, req, err)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("anydoc conversion failed for %q: %w", req.FileName, err)
|
||||
}
|
||||
if converted.AssetsError != nil {
|
||||
logger.Warnf(ctx, "[anydoc] %q parsed as text but its images were dropped: %v",
|
||||
req.FileName, converted.AssetsError)
|
||||
}
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: converted.Markdown,
|
||||
ImageRefs: imageRefsFromAssets(converted.Assets),
|
||||
Metadata: map[string]string{
|
||||
"parser": AnydocEngineName,
|
||||
"anydoc_version": anydoc.Version(),
|
||||
"source_format": format,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func pdfMarkdownIsEmpty(converted *anydoc.Result, err error) bool {
|
||||
if err != nil || converted == nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(converted.Markdown) == ""
|
||||
}
|
||||
|
||||
func (r *AnydocReader) readScannedPDF(ctx context.Context, req *types.ReadRequest, convertErr error) (*types.ReadResult, error) {
|
||||
if !r.canFallback() {
|
||||
if convertErr != nil {
|
||||
return nil, fmt.Errorf("anydoc conversion failed for %q: %w", req.FileName, convertErr)
|
||||
}
|
||||
return nil, fmt.Errorf("anydoc conversion failed for %q: PDF has no extractable text; OCR is required", req.FileName)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[anydoc] %q has no text layer, falling back to builtin for scanned-PDF OCR", req.FileName)
|
||||
fallbackReq := *req
|
||||
fallbackReq.ParserEngine = BuiltinEngineName
|
||||
result, err := r.fallback.Read(ctx, &fallbackReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("anydoc scanned-PDF fallback failed for %q: %w", req.FileName, err)
|
||||
}
|
||||
if result == nil {
|
||||
return nil, fmt.Errorf("anydoc scanned-PDF fallback returned no result for %q", req.FileName)
|
||||
}
|
||||
if result.Metadata == nil {
|
||||
result.Metadata = map[string]string{}
|
||||
}
|
||||
if result.Metadata["parser"] == "" {
|
||||
result.Metadata["parser"] = BuiltinEngineName
|
||||
}
|
||||
result.Metadata["anydoc_fallback"] = "scanned_pdf"
|
||||
if result.Metadata["image_source_type"] == "" {
|
||||
result.Metadata["image_source_type"] = "scanned_pdf"
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *AnydocReader) canFallback() bool {
|
||||
if r.fallback == nil {
|
||||
return false
|
||||
}
|
||||
if connected, ok := r.fallback.(interface{ IsConnected() bool }); ok {
|
||||
return connected.IsConnected()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// imageRefsFromAssets turns extracted assets into ImageRefs that match the
|
||||
// in-place markdown links renderMarkdown wrote.
|
||||
func imageRefsFromAssets(assets []anydoc.Asset) []types.ImageRef {
|
||||
if len(assets) == 0 {
|
||||
return nil
|
||||
}
|
||||
refs := make([]types.ImageRef, 0, len(assets))
|
||||
for _, asset := range assets {
|
||||
ref := anydoc.ImageDir + asset.Name
|
||||
refs = append(refs, types.ImageRef{
|
||||
Filename: asset.Name,
|
||||
OriginalRef: ref,
|
||||
MimeType: asset.MediaType,
|
||||
ImageData: asset.Data,
|
||||
})
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
// fileTypeOf reports the request's file type, falling back to the file name's
|
||||
// extension the way every reader in this package does.
|
||||
func fileTypeOf(req *types.ReadRequest) string {
|
||||
if ft := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(req.FileType)), "."); ft != "" {
|
||||
return ft
|
||||
}
|
||||
return strings.TrimPrefix(strings.ToLower(filepath.Ext(req.FileName)), ".")
|
||||
}
|
||||
|
||||
func isFalsey(value string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "false", "0", "no", "off":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestAnydocReaderRejectsUnsupportedFileType(t *testing.T) {
|
||||
_, err := NewAnydocReader(nil, nil).Read(context.Background(), &types.ReadRequest{
|
||||
FileName: "photo.png",
|
||||
FileType: "png",
|
||||
FileContent: []byte("not really a png"),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Read succeeded for an unsupported file type, want an error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "png") {
|
||||
t.Errorf("error does not name the file type: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnydocReaderRejectsURLs(t *testing.T) {
|
||||
_, err := NewAnydocReader(nil, nil).Read(context.Background(), &types.ReadRequest{
|
||||
URL: "https://example.com/report.docx",
|
||||
FileType: "docx",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Read succeeded for a URL request, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAnydocReaderHonoursImageExtractionOverride(t *testing.T) {
|
||||
if !NewAnydocReader(nil, nil).extractImages {
|
||||
t.Error("image extraction is off by default, want on")
|
||||
}
|
||||
if NewAnydocReader(map[string]string{"anydoc_extract_images": "false"}, nil).extractImages {
|
||||
t.Error("image extraction stayed on after the override turned it off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageRefsFromAssetsMatchInPlaceMarkdownPaths(t *testing.T) {
|
||||
refs := imageRefsFromAssets([]anydoc.Asset{
|
||||
{Name: "image-1.png", MediaType: "image/png", Data: []byte("first")},
|
||||
{Name: "image-2.jpg", MediaType: "image/jpeg", Data: []byte("second")},
|
||||
})
|
||||
if len(refs) != 2 {
|
||||
t.Fatalf("got %d image refs, want 2", len(refs))
|
||||
}
|
||||
if refs[0].OriginalRef != "images/image-1.png" {
|
||||
t.Errorf("first OriginalRef = %q, want images/image-1.png", refs[0].OriginalRef)
|
||||
}
|
||||
if refs[1].MimeType != "image/jpeg" {
|
||||
t.Errorf("second mime type = %q, want image/jpeg", refs[1].MimeType)
|
||||
}
|
||||
if len(refs[0].ImageData) == 0 {
|
||||
t.Error("image ref carries no bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageRefsFromAssetsEmpty(t *testing.T) {
|
||||
if refs := imageRefsFromAssets(nil); refs != nil {
|
||||
t.Errorf("got %d image refs, want none", len(refs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnydocReaderFallsBackForScannedPDF(t *testing.T) {
|
||||
fallback := &stubDocReader{result: &types.ReadResult{
|
||||
MarkdownContent: "",
|
||||
ImageRefs: []types.ImageRef{{
|
||||
Filename: "page_1.jpg",
|
||||
OriginalRef: "images/page_1.jpg",
|
||||
ImageData: []byte("jpeg"),
|
||||
}},
|
||||
Metadata: map[string]string{"image_source_type": "scanned_pdf"},
|
||||
}}
|
||||
result, err := NewAnydocReader(nil, fallback).readScannedPDF(context.Background(), &types.ReadRequest{
|
||||
FileName: "exam.pdf",
|
||||
FileType: "pdf",
|
||||
}, errors.New("PDF has no extractable text (Scanned, 5 pages): OCR is required"))
|
||||
if err != nil {
|
||||
t.Fatalf("readScannedPDF: %v", err)
|
||||
}
|
||||
if !strings.Contains(result.MarkdownContent, "images/page_1.jpg") {
|
||||
t.Errorf("fallback markdown missing page image:\n%s", result.MarkdownContent)
|
||||
}
|
||||
if result.Metadata["anydoc_fallback"] != "scanned_pdf" {
|
||||
t.Errorf("anydoc_fallback = %q, want scanned_pdf", result.Metadata["anydoc_fallback"])
|
||||
}
|
||||
if result.Metadata["image_source_type"] != "scanned_pdf" {
|
||||
t.Errorf("image_source_type = %q, want scanned_pdf", result.Metadata["image_source_type"])
|
||||
}
|
||||
if fallback.got == nil || fallback.got.ParserEngine != BuiltinEngineName {
|
||||
t.Errorf("fallback request engine = %v, want builtin", fallback.got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnydocReaderScannedPDFWithoutFallbackKeepsError(t *testing.T) {
|
||||
_, err := NewAnydocReader(nil, nil).readScannedPDF(context.Background(), &types.ReadRequest{
|
||||
FileName: "exam.pdf",
|
||||
}, errors.New("PDF has no extractable text: OCR is required"))
|
||||
if err == nil {
|
||||
t.Fatal("readScannedPDF succeeded without a fallback, want an error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "OCR is required") {
|
||||
t.Errorf("error does not mention OCR: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnydocReaderDisconnectedFallbackKeepsError(t *testing.T) {
|
||||
fallback := &disconnectedDocReader{}
|
||||
_, err := NewAnydocReader(nil, fallback).readScannedPDF(context.Background(), &types.ReadRequest{
|
||||
FileName: "exam.pdf",
|
||||
}, errors.New("PDF has no extractable text: OCR is required"))
|
||||
if err == nil {
|
||||
t.Fatal("readScannedPDF succeeded with a disconnected fallback, want an error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "OCR is required") {
|
||||
t.Errorf("error does not mention OCR: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type stubDocReader struct {
|
||||
got *types.ReadRequest
|
||||
result *types.ReadResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *stubDocReader) Read(_ context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
s.got = req
|
||||
return s.result, s.err
|
||||
}
|
||||
|
||||
type disconnectedDocReader struct{}
|
||||
|
||||
func (disconnectedDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) {
|
||||
return nil, errors.New("should not be called")
|
||||
}
|
||||
|
||||
func (disconnectedDocReader) IsConnected() bool { return false }
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
// Package docparser turns uploaded files and URLs into Markdown plus the
|
||||
// images they contain, which is the input the chunking and multimodal stages
|
||||
// downstream expect.
|
||||
//
|
||||
// A parse request names a parser engine, and every engine is one entry in the
|
||||
// registry:
|
||||
//
|
||||
// - engines.go declares the catalog — each engine's name, description,
|
||||
// supported file types, availability probe, and the reader that does the
|
||||
// work.
|
||||
// - engine_registry.go turns an engine name into a reader (NewReader) and
|
||||
// merges the local catalog with the engines the Python docreader reports
|
||||
// over its ListEngines RPC (ListAllEngines).
|
||||
//
|
||||
// Readers differ in where the parsing happens: in this process
|
||||
// (SimpleFormatReader, AnydocReader), in the docreader service over gRPC or
|
||||
// HTTP (GRPCDocumentReader, HTTPDocumentReader), or in a remote API (MinerU,
|
||||
// PaddleOCR-VL, WeKnora Cloud). They all return types.ReadResult, so the rest
|
||||
// of the package — image resolution and storage, table normalization — is
|
||||
// shared regardless of which engine ran.
|
||||
package docparser
|
||||
|
|
@ -1,182 +1,92 @@
|
|||
package docparser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// EngineRegistration is the interface every locally registered parser engine
|
||||
// must implement. Remote-only engines (e.g. markitdown) are discovered via
|
||||
// the docreader ListEngines RPC and do not need a local registration.
|
||||
// EngineRegistration is what every locally registered parser engine provides:
|
||||
// the metadata the engine list shows, and the reader that does the parsing.
|
||||
// Remote-only engines (e.g. markitdown) live in the Python docreader, are
|
||||
// discovered through its ListEngines RPC, and never register here — the
|
||||
// registry routes them to the docreader client by default.
|
||||
type EngineRegistration interface {
|
||||
Name() string
|
||||
Description() string
|
||||
FileTypes(docreaderConnected bool) []string
|
||||
CheckAvailable(docreaderConnected bool, overrides map[string]string) (available bool, reason string)
|
||||
// NewReader builds the reader for one parse request. Returning an error
|
||||
// means this engine cannot serve the request (missing credentials,
|
||||
// unreachable service); the caller reports it rather than silently
|
||||
// parsing with something else.
|
||||
NewReader(ctx context.Context, deps ReaderDeps) (interfaces.DocReader, error)
|
||||
}
|
||||
|
||||
// localEngines holds all locally registered parser engines.
|
||||
// ReaderDeps carries everything an engine may need to build its reader but
|
||||
// cannot construct itself: tenant configuration, tenant credentials, and the
|
||||
// shared docreader connection.
|
||||
type ReaderDeps struct {
|
||||
// Overrides holds tenant-level engine configuration (service endpoints,
|
||||
// API keys), as produced by ParserEngineConfig.ToOverridesMap.
|
||||
Overrides map[string]string
|
||||
// Remote is the docreader client. Nil when the service is not connected.
|
||||
Remote interfaces.DocReader
|
||||
// WeKnoraCloudCredentials resolves the tenant's WeKnora Cloud
|
||||
// credentials. It is a function rather than a value because resolving
|
||||
// them can hit the database, which most engines never need. Nil, or a
|
||||
// nil return, means the tenant has not configured them.
|
||||
WeKnoraCloudCredentials func(ctx context.Context) *types.WeKnoraCloudCredentials
|
||||
}
|
||||
|
||||
// localEngines holds all locally registered parser engines, in registration
|
||||
// order — which is also the order the engine list is shown in.
|
||||
var localEngines []EngineRegistration
|
||||
|
||||
// RegisterEngine adds an engine to the local registry. Called in init().
|
||||
// RegisterEngine adds an engine to the local registry. Called from init().
|
||||
func RegisterEngine(e EngineRegistration) {
|
||||
localEngines = append(localEngines, e)
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterEngine(&builtinEngine{})
|
||||
RegisterEngine(&simpleEngine{})
|
||||
RegisterEngine(&weKnoraCloudEngine{})
|
||||
RegisterEngine(&mineruEngine{})
|
||||
RegisterEngine(&mineruCloudEngine{})
|
||||
RegisterEngine(&paddleOCRVLEngine{})
|
||||
RegisterEngine(&paddleOCRVLCloudEngine{})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// builtin — DocReader-backed parser for complex document formats.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type builtinEngine struct{}
|
||||
|
||||
func (e *builtinEngine) Name() string { return "builtin" }
|
||||
func (e *builtinEngine) Description() string {
|
||||
return "DocReader built-in parser engine"
|
||||
}
|
||||
func (e *builtinEngine) FileTypes(_ bool) []string {
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "epub", "html", "htm", "mhtml", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
}
|
||||
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
|
||||
if docreaderConnected {
|
||||
return true, ""
|
||||
// lookupEngine returns the locally registered engine with this name.
|
||||
func lookupEngine(name string) (EngineRegistration, bool) {
|
||||
for _, engine := range localEngines {
|
||||
if engine.Name() == name {
|
||||
return engine, true
|
||||
}
|
||||
}
|
||||
return false, "DocReader service not connected"
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// SimpleEngineName is the engine name for Go-native simple format handling.
|
||||
const SimpleEngineName = "simple"
|
||||
|
||||
// WeKnoraCloudEngineName is the engine name for WeKnoraCloud-backed document parsing.
|
||||
const WeKnoraCloudEngineName = "weknoracloud"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// simple — Go handles md/txt/csv natively, no external service needed.
|
||||
// Distinct from docreader's "builtin" which uses Python libraries for
|
||||
// complex formats (docx, pdf, etc.).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type simpleEngine struct{}
|
||||
|
||||
func (e *simpleEngine) Name() string { return SimpleEngineName }
|
||||
func (e *simpleEngine) Description() string {
|
||||
return "Simple format & image parsing (no external service required)"
|
||||
}
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"}
|
||||
}
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type weKnoraCloudEngine struct{}
|
||||
|
||||
func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName }
|
||||
func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" }
|
||||
func (e *weKnoraCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"}
|
||||
}
|
||||
func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) {
|
||||
if overrides["weknoracloud_app_id"] != "" {
|
||||
return true, ""
|
||||
// NewReader builds the reader for an engine.
|
||||
//
|
||||
// An empty engine name means "no explicit choice": simple formats are handled
|
||||
// in Go and everything else goes to the docreader service. An unknown name is
|
||||
// routed to the docreader too, so engines that only exist in the Python
|
||||
// service keep working without a Go-side registration.
|
||||
func NewReader(
|
||||
ctx context.Context, engine, fileType string, isURL bool, deps ReaderDeps,
|
||||
) (interfaces.DocReader, error) {
|
||||
if registration, ok := lookupEngine(engine); ok {
|
||||
return registration.NewReader(ctx, deps)
|
||||
}
|
||||
return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up."
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru — Go-native, calls self-hosted MinerU API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruEngine struct{}
|
||||
|
||||
func (e *mineruEngine) Name() string { return "mineru" }
|
||||
func (e *mineruEngine) Description() string { return "MinerU self-hosted service" }
|
||||
func (e *mineruEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
func (e *mineruEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
endpoint := strings.TrimSpace(overrides["mineru_endpoint"])
|
||||
if endpoint == "" {
|
||||
return false, "MinerU service not configured"
|
||||
if engine == "" && !isURL && IsSimpleFormat(fileType) {
|
||||
return &SimpleFormatReader{}, nil
|
||||
}
|
||||
return PingMinerU(endpoint)
|
||||
return remoteReader(deps)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru_cloud — Go-native, calls MinerU Cloud API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruCloudEngine struct{}
|
||||
|
||||
func (e *mineruCloudEngine) Name() string { return "mineru_cloud" }
|
||||
func (e *mineruCloudEngine) Description() string { return "MinerU Cloud API" }
|
||||
func (e *mineruCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
func (e *mineruCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
apiKey := strings.TrimSpace(overrides["mineru_api_key"])
|
||||
if apiKey == "" {
|
||||
return false, "MinerU API Key not configured"
|
||||
// remoteReader returns the docreader client, or an error when the service is
|
||||
// not connected — a nil interface value here would panic at the call site.
|
||||
func remoteReader(deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
if deps.Remote == nil {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
return PingMinerUCloud(apiKey)
|
||||
return deps.Remote, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// paddleocr_vl — Go-native, calls a self-hosted PaddleOCR-VL pipeline service
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type paddleOCRVLEngine struct{}
|
||||
|
||||
func (e *paddleOCRVLEngine) Name() string { return "paddleocr_vl" }
|
||||
func (e *paddleOCRVLEngine) Description() string { return "PaddleOCR-VL self-hosted service" }
|
||||
func (e *paddleOCRVLEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"}
|
||||
}
|
||||
func (e *paddleOCRVLEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
endpoint := strings.TrimSpace(overrides["paddleocr_vl_endpoint"])
|
||||
if endpoint == "" {
|
||||
return false, "PaddleOCR-VL service not configured"
|
||||
}
|
||||
return PingPaddleOCRVL(endpoint)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// paddleocr_vl_cloud — Go-native, calls the PaddleOCR-VL AI Studio cloud API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type paddleOCRVLCloudEngine struct{}
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) Name() string { return "paddleocr_vl_cloud" }
|
||||
func (e *paddleOCRVLCloudEngine) Description() string { return "PaddleOCR-VL Cloud API" }
|
||||
func (e *paddleOCRVLCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"}
|
||||
}
|
||||
func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
token := strings.TrimSpace(overrides["paddleocr_vl_cloud_token"])
|
||||
if token == "" {
|
||||
return false, "PaddleOCR-VL Cloud Token not configured"
|
||||
}
|
||||
return PingPaddleOCRVLCloud(token)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ListAllEngines — merge local + remote
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ListAllEngines returns the merged engine list: locally registered engines
|
||||
// plus engines discovered from the remote docreader via ListEngines RPC.
|
||||
//
|
||||
|
|
@ -187,7 +97,9 @@ func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]str
|
|||
// authoritative for its own capabilities).
|
||||
// - Remote engines not present locally are appended as-is, enabling
|
||||
// auto-discovery of newly added docreader engines without Go changes.
|
||||
func ListAllEngines(docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo) []types.ParserEngineInfo {
|
||||
func ListAllEngines(
|
||||
docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo,
|
||||
) []types.ParserEngineInfo {
|
||||
remoteMap := make(map[string]types.ParserEngineInfo, len(remoteEngines))
|
||||
for _, re := range remoteEngines {
|
||||
remoteMap[re.Name] = re
|
||||
|
|
@ -231,3 +143,9 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote
|
|||
|
||||
return result
|
||||
}
|
||||
|
||||
// errEngineUnavailable reports an engine that is registered but cannot run for
|
||||
// this tenant or this build.
|
||||
func errEngineUnavailable(engine, reason string) error {
|
||||
return fmt.Errorf("parser engine %q is unavailable: %s", engine, reason)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// stubRemote stands in for the docreader client. NewReader only ever hands it
|
||||
// back, so the embedded interface is never called — and would panic if routing
|
||||
// ever sent a parse here by mistake.
|
||||
type stubRemote struct{ interfaces.DocReader }
|
||||
|
||||
func TestNewReaderRoutesByEngine(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote := &stubRemote{}
|
||||
deps := ReaderDeps{Remote: remote}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
engine string
|
||||
fileType string
|
||||
isURL bool
|
||||
want any
|
||||
}{
|
||||
{name: "simple engine", engine: SimpleEngineName, fileType: "md", want: &SimpleFormatReader{}},
|
||||
{name: "builtin engine goes remote", engine: BuiltinEngineName, fileType: "md", want: remote},
|
||||
{name: "unset engine handles simple formats in Go", fileType: "csv", want: &SimpleFormatReader{}},
|
||||
{name: "unset engine sends complex formats to docreader", fileType: "docx", want: remote},
|
||||
{name: "URLs always go to docreader", fileType: "md", isURL: true, want: remote},
|
||||
{name: "docreader-only engines fall through", engine: "markitdown", fileType: "docx", want: remote},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reader, err := NewReader(ctx, tc.engine, tc.fileType, tc.isURL, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("NewReader: %v", err)
|
||||
}
|
||||
if _, isSimple := tc.want.(*SimpleFormatReader); isSimple {
|
||||
if _, ok := reader.(*SimpleFormatReader); !ok {
|
||||
t.Fatalf("reader = %T, want *SimpleFormatReader", reader)
|
||||
}
|
||||
return
|
||||
}
|
||||
if reader != tc.want {
|
||||
t.Fatalf("reader = %T, want the docreader client", reader)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A disconnected docreader has to surface as an error, not as a nil reader the
|
||||
// caller would dereference.
|
||||
func TestNewReaderReportsDisconnectedDocReader(t *testing.T) {
|
||||
if _, err := NewReader(context.Background(), BuiltinEngineName, "docx", false, ReaderDeps{}); err == nil {
|
||||
t.Fatal("NewReader succeeded without a docreader connection, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewReaderRequiresWeKnoraCloudCredentials(t *testing.T) {
|
||||
_, err := NewReader(context.Background(), WeKnoraCloudEngineName, "docx", false, ReaderDeps{
|
||||
WeKnoraCloudCredentials: func(context.Context) *types.WeKnoraCloudCredentials { return nil },
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("NewReader succeeded without credentials, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
// The anydoc engine is only linked into builds tagged `anydoc`; everywhere
|
||||
// else it must be listed as unavailable and refuse to build a reader, so a
|
||||
// knowledge base configured for it fails loudly instead of parsing with
|
||||
// something else.
|
||||
func TestAnydocEngineFollowsBuildAvailability(t *testing.T) {
|
||||
reader, err := NewReader(context.Background(), AnydocEngineName, "docx", false, ReaderDeps{})
|
||||
|
||||
if anydoc.Available() {
|
||||
if err != nil {
|
||||
t.Fatalf("NewReader: %v", err)
|
||||
}
|
||||
if _, ok := reader.(*AnydocReader); !ok {
|
||||
t.Fatalf("reader = %T, want *AnydocReader", reader)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("NewReader succeeded without the converter linked in, want an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAllEnginesIncludesAnydoc(t *testing.T) {
|
||||
for _, engine := range ListAllEngines(true, nil, nil) {
|
||||
if engine.Name != AnydocEngineName {
|
||||
continue
|
||||
}
|
||||
if engine.Available != anydoc.Available() {
|
||||
t.Errorf("anydoc availability = %v, want %v", engine.Available, anydoc.Available())
|
||||
}
|
||||
if !engine.Available && engine.UnavailableReason == "" {
|
||||
t.Error("anydoc is unavailable without a reason to show")
|
||||
}
|
||||
if len(engine.FileTypes) == 0 {
|
||||
t.Error("anydoc lists no file types")
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("anydoc engine not found in the engine list")
|
||||
}
|
||||
|
|
@ -0,0 +1,271 @@
|
|||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// Engine names, as stored in knowledge base parser rules and shown in the UI.
|
||||
const (
|
||||
// BuiltinEngineName is the DocReader (Python) parser suite.
|
||||
BuiltinEngineName = "builtin"
|
||||
// SimpleEngineName is Go-native handling of text formats and images.
|
||||
SimpleEngineName = "simple"
|
||||
// AnydocEngineName is the in-process anydoc office-document converter.
|
||||
AnydocEngineName = "anydoc"
|
||||
// WeKnoraCloudEngineName is the hosted WeKnora Cloud document reader.
|
||||
WeKnoraCloudEngineName = "weknoracloud"
|
||||
// MinerUEngineName is a self-hosted MinerU service.
|
||||
MinerUEngineName = "mineru"
|
||||
// MinerUCloudEngineName is the MinerU Cloud API.
|
||||
MinerUCloudEngineName = "mineru_cloud"
|
||||
// PaddleOCRVLEngineName is a self-hosted PaddleOCR-VL pipeline.
|
||||
PaddleOCRVLEngineName = "paddleocr_vl"
|
||||
// PaddleOCRVLCloudEngineName is the PaddleOCR-VL AI Studio cloud API.
|
||||
PaddleOCRVLCloudEngineName = "paddleocr_vl_cloud"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterEngine(&builtinEngine{})
|
||||
RegisterEngine(&simpleEngine{})
|
||||
RegisterEngine(&anydocEngine{})
|
||||
RegisterEngine(&weKnoraCloudEngine{})
|
||||
RegisterEngine(&mineruEngine{})
|
||||
RegisterEngine(&mineruCloudEngine{})
|
||||
RegisterEngine(&paddleOCRVLEngine{})
|
||||
RegisterEngine(&paddleOCRVLCloudEngine{})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// builtin — DocReader-backed parser for complex document formats.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type builtinEngine struct{}
|
||||
|
||||
func (e *builtinEngine) Name() string { return BuiltinEngineName }
|
||||
|
||||
func (e *builtinEngine) Description() string { return "DocReader built-in parser engine" }
|
||||
|
||||
func (e *builtinEngine) FileTypes(_ bool) []string {
|
||||
return []string{
|
||||
"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "epub",
|
||||
"html", "htm", "mhtml",
|
||||
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp",
|
||||
"mp3", "wav", "m4a", "flac", "ogg",
|
||||
}
|
||||
}
|
||||
|
||||
func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) {
|
||||
if docreaderConnected {
|
||||
return true, ""
|
||||
}
|
||||
return false, "DocReader service not connected"
|
||||
}
|
||||
|
||||
// NewReader returns the docreader client. Selecting "builtin" explicitly means
|
||||
// the docreader is wanted even for formats the simple reader could handle.
|
||||
func (e *builtinEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
return remoteReader(deps)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// simple — Go handles md/txt/csv/json natively, no external service needed.
|
||||
// Distinct from docreader's "builtin", which uses Python libraries for complex
|
||||
// formats (docx, pdf, ...).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type simpleEngine struct{}
|
||||
|
||||
func (e *simpleEngine) Name() string { return SimpleEngineName }
|
||||
|
||||
func (e *simpleEngine) Description() string {
|
||||
return "Simple format & image parsing (no external service required)"
|
||||
}
|
||||
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{
|
||||
"md", "markdown", "txt", "csv", "json",
|
||||
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp",
|
||||
"mp3", "wav", "m4a", "flac", "ogg",
|
||||
}
|
||||
}
|
||||
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func (e *simpleEngine) NewReader(_ context.Context, _ ReaderDeps) (interfaces.DocReader, error) {
|
||||
return &SimpleFormatReader{}, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// anydoc — office documents converted in this process, no external service.
|
||||
// Only present in builds that link the converter (see the anydoc package).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type anydocEngine struct{}
|
||||
|
||||
func (e *anydocEngine) Name() string { return AnydocEngineName }
|
||||
|
||||
func (e *anydocEngine) Description() string {
|
||||
return "anydoc in-process office document converter (no external service required)"
|
||||
}
|
||||
|
||||
func (e *anydocEngine) FileTypes(_ bool) []string { return anydoc.SupportedFileTypes() }
|
||||
|
||||
func (e *anydocEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
if anydoc.Available() {
|
||||
return true, ""
|
||||
}
|
||||
return false, anydoc.UnavailableReason()
|
||||
}
|
||||
|
||||
func (e *anydocEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
if !anydoc.Available() {
|
||||
return nil, errEngineUnavailable(AnydocEngineName, anydoc.UnavailableReason())
|
||||
}
|
||||
return NewAnydocReader(deps.Overrides, deps.Remote), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type weKnoraCloudEngine struct{}
|
||||
|
||||
func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName }
|
||||
|
||||
func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" }
|
||||
|
||||
func (e *weKnoraCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"}
|
||||
}
|
||||
|
||||
func (e *weKnoraCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
if overrides["weknoracloud_app_id"] != "" {
|
||||
return true, ""
|
||||
}
|
||||
return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up."
|
||||
}
|
||||
|
||||
func (e *weKnoraCloudEngine) NewReader(
|
||||
ctx context.Context, deps ReaderDeps,
|
||||
) (interfaces.DocReader, error) {
|
||||
if deps.WeKnoraCloudCredentials == nil {
|
||||
return nil, errEngineUnavailable(WeKnoraCloudEngineName, "no credential resolver configured")
|
||||
}
|
||||
creds := deps.WeKnoraCloudCredentials(ctx)
|
||||
if creds == nil {
|
||||
return nil, errEngineUnavailable(WeKnoraCloudEngineName, "tenant credentials not configured")
|
||||
}
|
||||
return NewWeKnoraCloudSignedDocumentReader(creds.AppID, creds.AppSecret)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru — Go-native, calls self-hosted MinerU API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruEngine struct{}
|
||||
|
||||
func (e *mineruEngine) Name() string { return MinerUEngineName }
|
||||
|
||||
func (e *mineruEngine) Description() string { return "MinerU self-hosted service" }
|
||||
|
||||
func (e *mineruEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
|
||||
func (e *mineruEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
endpoint := strings.TrimSpace(overrides["mineru_endpoint"])
|
||||
if endpoint == "" {
|
||||
return false, "MinerU service not configured"
|
||||
}
|
||||
return PingMinerU(endpoint)
|
||||
}
|
||||
|
||||
func (e *mineruEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
return NewMinerUReader(deps.Overrides), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru_cloud — Go-native, calls MinerU Cloud API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruCloudEngine struct{}
|
||||
|
||||
func (e *mineruCloudEngine) Name() string { return MinerUCloudEngineName }
|
||||
|
||||
func (e *mineruCloudEngine) Description() string { return "MinerU Cloud API" }
|
||||
|
||||
func (e *mineruCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
|
||||
func (e *mineruCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
apiKey := strings.TrimSpace(overrides["mineru_api_key"])
|
||||
if apiKey == "" {
|
||||
return false, "MinerU API Key not configured"
|
||||
}
|
||||
return PingMinerUCloud(apiKey)
|
||||
}
|
||||
|
||||
func (e *mineruCloudEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
return NewMinerUCloudReader(deps.Overrides), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// paddleocr_vl — Go-native, calls a self-hosted PaddleOCR-VL pipeline service
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type paddleOCRVLEngine struct{}
|
||||
|
||||
func (e *paddleOCRVLEngine) Name() string { return PaddleOCRVLEngineName }
|
||||
|
||||
func (e *paddleOCRVLEngine) Description() string { return "PaddleOCR-VL self-hosted service" }
|
||||
|
||||
func (e *paddleOCRVLEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"}
|
||||
}
|
||||
|
||||
func (e *paddleOCRVLEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
endpoint := strings.TrimSpace(overrides["paddleocr_vl_endpoint"])
|
||||
if endpoint == "" {
|
||||
return false, "PaddleOCR-VL service not configured"
|
||||
}
|
||||
return PingPaddleOCRVL(endpoint)
|
||||
}
|
||||
|
||||
func (e *paddleOCRVLEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) {
|
||||
return NewPaddleOCRVLReader(deps.Overrides), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// paddleocr_vl_cloud — Go-native, calls the PaddleOCR-VL AI Studio cloud API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type paddleOCRVLCloudEngine struct{}
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) Name() string { return PaddleOCRVLCloudEngineName }
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) Description() string { return "PaddleOCR-VL Cloud API" }
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"}
|
||||
}
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
token := strings.TrimSpace(overrides["paddleocr_vl_cloud_token"])
|
||||
if token == "" {
|
||||
return false, "PaddleOCR-VL Cloud Token not configured"
|
||||
}
|
||||
return PingPaddleOCRVLCloud(token)
|
||||
}
|
||||
|
||||
func (e *paddleOCRVLCloudEngine) NewReader(
|
||||
_ context.Context, deps ReaderDeps,
|
||||
) (interfaces.DocReader, error) {
|
||||
return NewPaddleOCRVLCloudReader(deps.Overrides), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
#!/usr/bin/env bash
|
||||
# Build the anydoc static archive that the `anydoc` build tag links.
|
||||
#
|
||||
# The archive is a Rust build artifact (~30 MB), so it is neither committed nor
|
||||
# downloaded: this script builds it from third_party/anydoc-go, which pins the
|
||||
# published anydoc crate, and drops it where cgo looks for it.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-anydoc-lib.sh # host platform
|
||||
# TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
crate_dir="$repo_root/third_party/anydoc-go"
|
||||
|
||||
if ! command -v cargo >/dev/null 2>&1; then
|
||||
echo "error: cargo not found. Install Rust (https://rustup.rs) to build the anydoc archive." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The Rust target triple decides the archive; the directory name mirrors what
|
||||
# the cgo LDFLAGS in third_party/anydoc-go expect for that platform.
|
||||
target=${TARGET:-$(rustc -vV | sed -n 's/^host: //p')}
|
||||
case "$target" in
|
||||
x86_64-apple-darwin) lib_dir=darwin_amd64 ;;
|
||||
aarch64-apple-darwin) lib_dir=darwin_arm64 ;;
|
||||
x86_64-pc-windows-msvc) lib_dir=windows_amd64 ;;
|
||||
x86_64-unknown-linux-gnu) lib_dir=linux_amd64_gnu ;;
|
||||
aarch64-unknown-linux-gnu) lib_dir=linux_arm64_gnu ;;
|
||||
x86_64-unknown-linux-musl) lib_dir=linux_amd64_musl ;;
|
||||
aarch64-unknown-linux-musl) lib_dir=linux_arm64_musl ;;
|
||||
*)
|
||||
echo "error: unsupported target '$target'." >&2
|
||||
echo "Supported: {x86_64,aarch64}-{apple-darwin,unknown-linux-{gnu,musl}}, x86_64-pc-windows-msvc" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$target" in
|
||||
*windows-msvc) lib_name=anydoc_go.lib ;;
|
||||
*) lib_name=libanydoc_go.a ;;
|
||||
esac
|
||||
|
||||
# document_to_markdown is private in published anydoc. Copy 0.1.8 and re-export
|
||||
# that one function so anydoc-go can keep the official serializer.
|
||||
prepare_patched_anydoc() {
|
||||
local dest="$crate_dir/patched-anydoc"
|
||||
if [ -f "$dest/.weknora-patched" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local src=""
|
||||
local cargo_home="${CARGO_HOME:-$HOME/.cargo}"
|
||||
src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name 'anydoc-0.1.8' 2>/dev/null | head -1 || true)
|
||||
|
||||
if [ -z "$src" ]; then
|
||||
local tarball="$crate_dir/.anydoc-0.1.8.crate"
|
||||
echo "Fetching anydoc 0.1.8 to patch document_to_markdown into the public API"
|
||||
if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/0.1.8/download" -o "$tarball"; then
|
||||
curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-0.1.8.crate" -o "$tarball"
|
||||
fi
|
||||
local unpack="$crate_dir/.anydoc-unpack"
|
||||
rm -rf "$unpack"
|
||||
mkdir -p "$unpack"
|
||||
tar -xzf "$tarball" -C "$unpack"
|
||||
src=$(find "$unpack" -maxdepth 1 -type d -name 'anydoc-0.1.8' | head -1)
|
||||
if [ -z "$src" ]; then
|
||||
echo "error: unpacked anydoc-0.1.8 crate is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -rf "$dest"
|
||||
cp -R "$src" "$dest"
|
||||
if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then
|
||||
echo "error: anydoc 0.1.8 lib.rs no longer has the expected document_to_markdown import" >&2
|
||||
exit 1
|
||||
fi
|
||||
sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs"
|
||||
rm -f "$dest/src/lib.rs.bak"
|
||||
touch "$dest/.weknora-patched"
|
||||
rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-0.1.8.crate"
|
||||
}
|
||||
|
||||
# --locked: build exactly the dependency versions in the committed Cargo.lock.
|
||||
# That lockfile is what pins the patched pdf-inspector/lopdf, so a silent
|
||||
# resolver drift must fail the build rather than ship an unaudited tree.
|
||||
echo "Building anydoc archive for $target"
|
||||
prepare_patched_anydoc
|
||||
cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target"
|
||||
|
||||
dest="$crate_dir/lib/$lib_dir"
|
||||
mkdir -p "$dest"
|
||||
cp "$crate_dir/target/$target/release/$lib_name" "$dest/$lib_name"
|
||||
|
||||
echo "Wrote $dest/$lib_name"
|
||||
echo "Build WeKnora with the archive: go build -tags anydoc ./cmd/server"
|
||||
|
|
@ -406,6 +406,46 @@ check_remote_dev_connectivity() {
|
|||
return 0
|
||||
}
|
||||
|
||||
# Host-platform path of the anydoc static archive (built by `make anydoc-lib`).
|
||||
anydoc_host_archive() {
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_arm64/libanydoc_go.a" ;;
|
||||
Darwin-x86_64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_amd64/libanydoc_go.a" ;;
|
||||
Linux-x86_64)
|
||||
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a" ]; then
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a"
|
||||
else
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_musl/libanydoc_go.a"
|
||||
fi
|
||||
;;
|
||||
Linux-aarch64)
|
||||
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a" ]; then
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a"
|
||||
else
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_musl/libanydoc_go.a"
|
||||
fi
|
||||
;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Enable the in-process anydoc engine when the archive is present, unless the
|
||||
# caller already set GO_BUILD_TAGS (including empty, which opts out).
|
||||
enable_anydoc_build_tag() {
|
||||
if [ -n "${GO_BUILD_TAGS+x}" ]; then
|
||||
export GO_BUILD_TAGS
|
||||
return
|
||||
fi
|
||||
local archive
|
||||
archive="$(anydoc_host_archive)"
|
||||
if [ -n "$archive" ] && [ -f "$archive" ]; then
|
||||
export GO_BUILD_TAGS=anydoc
|
||||
log_info "检测到 anydoc 静态库,已启用 -tags anydoc"
|
||||
else
|
||||
log_info "未检测到 anydoc 静态库,解析引擎不可用。需要时先运行: make anydoc-lib"
|
||||
fi
|
||||
}
|
||||
|
||||
# 启动后端应用(本地)
|
||||
start_app() {
|
||||
log_info "启动后端应用(本地开发模式)..."
|
||||
|
|
@ -477,6 +517,8 @@ start_app() {
|
|||
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
|
||||
fi
|
||||
|
||||
enable_anydoc_build_tag
|
||||
|
||||
# 检查是否安装了 Air(热重载工具)
|
||||
if command -v air &> /dev/null; then
|
||||
log_success "检测到 Air,使用热重载模式启动..."
|
||||
|
|
@ -487,7 +529,7 @@ start_app() {
|
|||
log_warning "提示: 安装 Air 可以实现代码修改后自动重启"
|
||||
log_info "安装命令: go install github.com/air-verse/air@latest"
|
||||
LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"
|
||||
go run -ldflags="$LDFLAGS" ./cmd/server
|
||||
go run -tags "${GO_BUILD_TAGS:-}" -ldflags="$LDFLAGS" ./cmd/server
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
target/
|
||||
lib/
|
||||
patched-anydoc/
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,39 @@
|
|||
# Go bindings for anydoc.
|
||||
#
|
||||
# `crate-type = ["staticlib"]` produces libanydoc_go.a, which the Go module
|
||||
# links at build time via cgo. Not a cdylib: cgo wants an archive it can pull
|
||||
# symbols from, not a loadable shared object.
|
||||
[package]
|
||||
name = "anydoc-go"
|
||||
version = "0.1.8"
|
||||
edition = "2024"
|
||||
description = "Go bindings (C ABI) for anydoc"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/firecrawl/anydoc"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
crate-type = ["staticlib"]
|
||||
|
||||
# Standalone crate: it lives inside the WeKnora tree, not the anydoc
|
||||
# workspace, so it takes the published crate rather than a path dependency.
|
||||
# Pinned exactly: the C ABI mirrors this version's error and format enums.
|
||||
[dependencies]
|
||||
anydoc = "=0.1.8"
|
||||
|
||||
[build-dependencies]
|
||||
cbindgen = "0.29"
|
||||
|
||||
# document_to_markdown is crate-private in published anydoc. The build script
|
||||
# copies 0.1.8 and re-exports that one function so we can keep the official
|
||||
# GFM serializer after rewriting asset images to External URLs.
|
||||
[patch.crates-io]
|
||||
anydoc = { path = "patched-anydoc" }
|
||||
|
||||
[workspace]
|
||||
|
||||
# Mirrors the anydoc workspace's release profile: without it the archive keeps
|
||||
# every debug symbol and roughly doubles in size.
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
strip = "symbols"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2026 Sideguide Technologies Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
# anydoc-go (vendored)
|
||||
|
||||
Go bindings for [anydoc](https://github.com/firecrawl/anydoc), the Rust library
|
||||
that converts Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF
|
||||
documents to GitHub-Flavored Markdown. WeKnora links them through cgo so office
|
||||
documents can be parsed inside the Go process, without the Python docreader
|
||||
service.
|
||||
|
||||
## Why this is vendored
|
||||
|
||||
The bindings come from the open pull request
|
||||
[firecrawl/anydoc#30](https://github.com/firecrawl/anydoc/pull/30). Until it
|
||||
merges and upstream tags `go/vX.Y.Z`, there is no module to `go get`: the PR
|
||||
also expects a maintainer-only workflow to commit per-platform archives that do
|
||||
not exist yet. So the binding source lives here and `go.mod` points at it:
|
||||
|
||||
```
|
||||
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
|
||||
```
|
||||
|
||||
When the upstream module is published, delete this directory, drop the
|
||||
`replace`, and require the published version. Nothing else in WeKnora changes:
|
||||
only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.
|
||||
|
||||
## Provenance
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") |
|
||||
| PR head | `1a7a6c0` |
|
||||
| Rebased onto | `4e3089b` (`chore: release v0.1.8`) |
|
||||
| anydoc crate | `0.1.8` (from crates.io, pinned with `=`) |
|
||||
| License | MIT (see LICENSE) |
|
||||
|
||||
The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before
|
||||
vendoring. The rebase conflicts were all release bookkeeping — the workspace
|
||||
member list, the README binding sections, and the version-agreement script,
|
||||
which the wasm bindings had touched in the meantime — plus the binding's own
|
||||
version, bumped from 0.1.3 to 0.1.8. Taking v0.1.8 also picks up the
|
||||
`pdf-inspector` 0.1.8 bump that fixes [RUSTSEC-2026-0187](https://rustsec.org/advisories/RUSTSEC-2026-0187.html),
|
||||
the `lopdf` stack overflow that aborts the process on a hostile PDF.
|
||||
|
||||
## Local modifications
|
||||
|
||||
Keep this list current: it is the diff a future upgrade has to re-apply. Items
|
||||
2–4 are bugs in the upstream PR and are worth sending back to it.
|
||||
|
||||
1. `Cargo.toml` — depends on the published `anydoc = "=0.1.8"` crate instead of
|
||||
the workspace path dependency, declares its own empty `[workspace]`, and
|
||||
repeats the upstream release profile (`lto`, `strip`), which it would
|
||||
otherwise inherit from the anydoc workspace.
|
||||
2. `anydoc.go` — every ABI call runs inside `call()`, which pins the goroutine
|
||||
with `runtime.LockOSThread` for the duration. The ABI reports the error
|
||||
message through a thread-local slot that a *second* call
|
||||
(`anydoc_last_error`) reads, and Go may resume a goroutine on a different OS
|
||||
thread once a cgo call returns: without pinning, a failed conversion reports
|
||||
an empty message, or one belonging to another document parsed on the thread
|
||||
it landed on. Reproduced at roughly 1 in 1500 concurrent conversions; the
|
||||
regression test is `TestErrorDetailSurvivesConcurrency` in
|
||||
`internal/infrastructure/docparser/anydoc`.
|
||||
3. `model.go` — decoder preallocations are bounded by the bytes left in the
|
||||
buffer (`capFor`), and `need` rejects a negative length. A count taken
|
||||
straight from the buffer is only trustworthy while the Rust encoder and this
|
||||
decoder agree; on a skew, `make([]Block, 0, n)` would exhaust memory before
|
||||
the first bounds check, turning a version mismatch into a dead process.
|
||||
4. `src/lib.rs` — the three conversion entry points run inside `guarded()`,
|
||||
which catches a panic and reports it as a malformed document. A panic
|
||||
escaping an `extern "C"` function aborts the process, and WeKnora parses
|
||||
untrusted uploads in the same process that serves the API. Note the limit:
|
||||
this cannot contain a stack overflow or an allocation failure, which is why
|
||||
the dependency pin below matters as much as the guard.
|
||||
5. Removed the upstream CLI (`cmd/anydoc`) and the binding test suite, which
|
||||
reads fixtures from the anydoc repository. WeKnora's own tests live in
|
||||
`internal/infrastructure/docparser/anydoc`.
|
||||
6. `scripts/build-anydoc-lib.sh` copies anydoc 0.1.8 to `patched-anydoc/`
|
||||
(gitignored) and re-exports `document_to_markdown`, which is crate-private
|
||||
in the published crate. `src/asset_links.rs` then rewrites `ImageSource::Asset`
|
||||
to `External("images/image-N.ext")` so the official serializer emits in-place
|
||||
image links. `anydoc_to_markdown_with_asset_links` is the ABI for that path.
|
||||
|
||||
## Dependency pinning and audit
|
||||
|
||||
`Cargo.lock` is committed and `scripts/build-anydoc-lib.sh` builds with
|
||||
`--locked`, so the archive is always the audited dependency tree. CI runs
|
||||
`cargo audit` against it, because the crate that fails here is the one parsing
|
||||
untrusted uploads inside the API process.
|
||||
|
||||
That matters concretely: with `lopdf` 0.41 — what the upstream PR's own
|
||||
lockfile resolved to — a ~100 KB PDF holding a deeply nested catalog array
|
||||
kills the process with a stack overflow (`RUSTSEC-2026-0187`), which neither
|
||||
`guarded()` nor Go's `recover` can contain. anydoc 0.1.8 moved to `lopdf` 0.42
|
||||
and the same input comes back as an ordinary error;
|
||||
`TestDeeplyNestedPDFFailsWithoutKillingTheProcess` keeps it that way.
|
||||
|
||||
`cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is
|
||||
unmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. It
|
||||
is not a vulnerability and nothing here can fix it, so warnings report without
|
||||
failing the job.
|
||||
|
||||
## Known upstream limitation
|
||||
|
||||
Markdown rendering drops embedded images: `ImageSource::Asset` renders as its
|
||||
alt text, and the bytes are only reachable through the document model. The
|
||||
renderer (`document_to_markdown`) is also crate-private. WeKnora keeps that
|
||||
serializer: `scripts/build-anydoc-lib.sh` re-exports the one function, then
|
||||
`anydoc_to_markdown_with_asset_links` rewrites `Asset` images to
|
||||
`ImageSource::External("images/image-N.ext")` so the official GFM output
|
||||
places them in reading order. PDF has no document model; scanned pages fall
|
||||
back to the builtin docreader so they can be rasterized for OCR.
|
||||
|
||||
## Building the archive
|
||||
|
||||
cgo links `lib/<platform>/libanydoc_go.a`, which is a build artifact (~30 MB)
|
||||
and is therefore git-ignored rather than committed:
|
||||
|
||||
```bash
|
||||
scripts/build-anydoc-lib.sh # host platform
|
||||
TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh
|
||||
```
|
||||
|
||||
Then build WeKnora with the engine linked in:
|
||||
|
||||
```bash
|
||||
make build-anydoc # or: go build -tags anydoc ./cmd/server
|
||||
```
|
||||
|
||||
Builds without the `anydoc` tag need no Rust toolchain and no archive; the
|
||||
engine simply reports itself as unavailable.
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
// Package anydoc converts documents (Word, PowerPoint, Excel, OpenDocument,
|
||||
// RTF, EPUB, CSV, and PDF) to GitHub-Flavored Markdown, with full access to
|
||||
// the parsed document model and embedded assets.
|
||||
//
|
||||
// This is the Go binding. It links a Rust static library through cgo; the
|
||||
// module packages archives for every supported platform. See the README for
|
||||
// the supported platform matrix and musl build tag.
|
||||
package anydoc
|
||||
|
||||
/*
|
||||
#cgo CFLAGS: -I${SRCDIR}/include
|
||||
#cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/lib/darwin_arm64 -lanydoc_go -lm -lstdc++
|
||||
#cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/lib/darwin_amd64 -lanydoc_go -lm -lstdc++
|
||||
#cgo windows,amd64 LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -lanydoc_go
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "include/anydoc.h"
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// call runs one ABI entry point and turns its status code into an error.
|
||||
//
|
||||
// The goroutine is pinned for the duration because the ABI reports the error
|
||||
// message through a thread-local slot that a *second* call
|
||||
// (`anydoc_last_error`) reads. Go is free to resume a goroutine on a different
|
||||
// OS thread after a cgo call returns, and a conversion is long enough for that
|
||||
// to happen: the message would then be missing, or belong to another
|
||||
// conversion that ran on the thread we landed on.
|
||||
func call(entry func() C.int) error {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
if code := entry(); code != errOK {
|
||||
return convertError(code)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ToMarkdown converts a document file to Markdown. The format is detected
|
||||
// from the file content; the extension is the fallback for signature-less
|
||||
// formats (CSV) and unrecognizable containers.
|
||||
func ToMarkdown(path string) (string, error) {
|
||||
cpath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cpath))
|
||||
var out *C.char
|
||||
var outLen C.uintptr_t
|
||||
if err := call(func() C.int { return C.anydoc_to_markdown(cpath, &out, &outLen) }); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer C.anydoc_string_free(out)
|
||||
markdown, err := cStringN(out, outLen)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return markdown, nil
|
||||
}
|
||||
|
||||
// ToMarkdownBytes converts an in-memory document to Markdown. Pass a Format
|
||||
// to select the parser, or nil to detect it from the content, which
|
||||
// signature-less formats (CSV) have to name explicitly.
|
||||
func ToMarkdownBytes(data []byte, format *Format) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", &ConvertError{Kind: "unsupported", Detail: "empty input"}
|
||||
}
|
||||
tag := C.int(C.ANYDOC_FORMAT_NONE)
|
||||
if format != nil {
|
||||
tag = formatToTag(*format)
|
||||
if tag == C.int(C.ANYDOC_FORMAT_NONE) {
|
||||
return "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)}
|
||||
}
|
||||
}
|
||||
var out *C.char
|
||||
var outLen C.uintptr_t
|
||||
if err := call(func() C.int {
|
||||
return C.anydoc_to_markdown_bytes(
|
||||
(*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &out, &outLen,
|
||||
)
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer C.anydoc_string_free(out)
|
||||
markdown, err := cStringN(out, outLen)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return markdown, nil
|
||||
}
|
||||
|
||||
// ToMarkdownWithAssetLinks converts an in-memory document to Markdown with
|
||||
// embedded images rewritten as `` so they keep
|
||||
// their original positions. The Markdown itself is produced by anydoc's
|
||||
// official GFM serializer.
|
||||
func ToMarkdownWithAssetLinks(data []byte, format *Format) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", &ConvertError{Kind: "unsupported", Detail: "empty input"}
|
||||
}
|
||||
tag := C.int(C.ANYDOC_FORMAT_NONE)
|
||||
if format != nil {
|
||||
tag = formatToTag(*format)
|
||||
if tag == C.int(C.ANYDOC_FORMAT_NONE) {
|
||||
return "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)}
|
||||
}
|
||||
}
|
||||
var out *C.char
|
||||
var outLen C.uintptr_t
|
||||
if err := call(func() C.int {
|
||||
return C.anydoc_to_markdown_with_asset_links(
|
||||
(*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &out, &outLen,
|
||||
)
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer C.anydoc_string_free(out)
|
||||
markdown, err := cStringN(out, outLen)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return markdown, nil
|
||||
}
|
||||
|
||||
// ToDocument parses an in-memory document into the document model, which also
|
||||
// carries the embedded assets. Pass a Format to select the parser, or nil to
|
||||
// detect it from the content.
|
||||
//
|
||||
// Unsupported for PDF: PDF conversion produces Markdown directly and has no
|
||||
// document-model form; use ToMarkdownBytes.
|
||||
func ToDocument(data []byte, format *Format) (*Document, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, &ConvertError{Kind: "unsupported", Detail: "empty input"}
|
||||
}
|
||||
tag := C.int(C.ANYDOC_FORMAT_NONE)
|
||||
if format != nil {
|
||||
tag = formatToTag(*format)
|
||||
if tag == C.int(C.ANYDOC_FORMAT_NONE) {
|
||||
return nil, &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)}
|
||||
}
|
||||
}
|
||||
var buf *C.uint8_t
|
||||
var bufLen C.uintptr_t
|
||||
if err := call(func() C.int {
|
||||
return C.anydoc_to_document(
|
||||
(*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &buf, &bufLen,
|
||||
)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer C.anydoc_buffer_free(buf, bufLen)
|
||||
if buf == nil || bufLen == 0 {
|
||||
return &Document{}, nil
|
||||
}
|
||||
// Copy the C buffer into a Go-owned slice before decoding, so the C
|
||||
// buffer can be freed immediately rather than tying its lifetime to the
|
||||
// decoded Document.
|
||||
raw, err := cGoBytes(unsafe.Pointer(buf), bufLen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeDocument(raw)
|
||||
}
|
||||
|
||||
// cStringN reads a length-prefixed C buffer as a Go string. The bytes are
|
||||
// copied, so the caller may free the C buffer immediately after.
|
||||
func cStringN(s *C.char, n C.uintptr_t) (string, error) {
|
||||
if s == nil || n == 0 {
|
||||
return "", nil
|
||||
}
|
||||
b, err := cGoBytes(unsafe.Pointer(s), n)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func cGoBytes(ptr unsafe.Pointer, n C.uintptr_t) ([]byte, error) {
|
||||
// C.GoBytes accepts a C int length even on 64-bit Go builds.
|
||||
if uint64(n) > 1<<31-1 {
|
||||
return nil, &ConvertError{Kind: "resource_limit", Detail: fmt.Sprintf("FFI output is too large for Go: %d bytes", n)}
|
||||
}
|
||||
return C.GoBytes(ptr, C.int(n)), nil
|
||||
}
|
||||
|
||||
// cString reads a NUL-terminated C string as a Go string.
|
||||
func cString(s *C.char) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return C.GoString(s)
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
//! Regenerate `include/anydoc.h` from the crate's C ABI on every build.
|
||||
//! `cbindgen` is a build-dependency, so this runs whenever `cargo build
|
||||
//! -p anydoc-go` runs inside the workspace. The committed header is the
|
||||
//! source of truth for the Go side; consumers link the packaged static archive
|
||||
//! and never run this.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
|
||||
let include_dir = manifest_dir.join("include");
|
||||
let header = include_dir.join("anydoc.h");
|
||||
|
||||
std::fs::create_dir_all(&include_dir).expect("create include directory");
|
||||
|
||||
let config = cbindgen::Config {
|
||||
language: cbindgen::Language::C,
|
||||
include_guard: Some("ANYDOC_H".to_string()),
|
||||
autogen_warning: Some(
|
||||
"/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */"
|
||||
.to_string(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match cbindgen::Builder::new().with_crate(&manifest_dir).with_config(config).generate() {
|
||||
Ok(bindings) => {
|
||||
let _ = bindings.write_to_file(&header);
|
||||
}
|
||||
Err(err) => {
|
||||
println!("cargo:warning=cbindgen failed to generate anydoc.h: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
println!("cargo:rerun-if-changed=src/lib.rs");
|
||||
println!("cargo:rerun-if-changed=src/model.rs");
|
||||
println!("cargo:rerun-if-changed=src/asset_links.rs");
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package anydoc
|
||||
|
||||
/*
|
||||
#include <stdint.h>
|
||||
#include "include/anydoc.h"
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Error codes mirroring the C ABI (ERR_*). Stable across versions.
|
||||
const (
|
||||
errOK = C.ERR_OK
|
||||
errUnsupported = C.ERR_UNSUPPORTED
|
||||
errMalformed = C.ERR_MALFORMED
|
||||
errEncrypted = C.ERR_ENCRYPTED
|
||||
errResourceLimit = C.ERR_RESOURCE_LIMIT
|
||||
errMissingPart = C.ERR_MISSING_PART
|
||||
errIO = C.ERR_IO
|
||||
errPDFNoModel = C.ERR_PDF_NO_MODEL
|
||||
errInvalidArg = C.ERR_INVALID_ARG
|
||||
errUnknownFormat = C.ERR_UNKNOWN_FORMAT
|
||||
)
|
||||
|
||||
// ConvertError is the typed error every conversion function returns. It
|
||||
// carries the same variant kind the Node and Python bindings expose
|
||||
// ("unsupported", "malformed", "encrypted", ...) plus the crate's
|
||||
// human-readable detail.
|
||||
type ConvertError struct {
|
||||
// Kind is the lowercase variant name, matching the Node and Python
|
||||
// bindings: "unsupported", "malformed", "encrypted", "resource_limit",
|
||||
// "missing_part", "io", "pdf_no_model". Go also reports
|
||||
// "unknown_format" for an invalid explicit Format.
|
||||
Kind string
|
||||
// Detail is the crate's Display output for the error.
|
||||
Detail string
|
||||
}
|
||||
|
||||
func (e *ConvertError) Error() string {
|
||||
if e == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
if e.Detail == "" {
|
||||
return "anydoc: " + e.Kind
|
||||
}
|
||||
return "anydoc: " + e.Kind + ": " + e.Detail
|
||||
}
|
||||
|
||||
// convertError builds a ConvertError from an ABI error code, pulling the
|
||||
// detail message from the thread-local last-error slot.
|
||||
func convertError(code C.int) error {
|
||||
if code == errOK {
|
||||
return nil
|
||||
}
|
||||
kind := errorKind(code)
|
||||
detail := lastError()
|
||||
return &ConvertError{Kind: kind, Detail: detail}
|
||||
}
|
||||
|
||||
func errorKind(code C.int) string {
|
||||
switch code {
|
||||
case errUnsupported:
|
||||
return "unsupported"
|
||||
case errMalformed:
|
||||
return "malformed"
|
||||
case errEncrypted:
|
||||
return "encrypted"
|
||||
case errResourceLimit:
|
||||
return "resource_limit"
|
||||
case errMissingPart:
|
||||
return "missing_part"
|
||||
case errIO:
|
||||
return "io"
|
||||
case errPDFNoModel:
|
||||
return "pdf_no_model"
|
||||
case errInvalidArg:
|
||||
return "invalid_argument"
|
||||
case errUnknownFormat:
|
||||
return "unknown_format"
|
||||
default:
|
||||
return fmt.Sprintf("unknown_error(%d)", int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// lastError returns the human-readable message the last ABI call stashed on
|
||||
// the current OS thread, or "" when there was none. The C side returns a
|
||||
// freshly allocated string the caller must free.
|
||||
func lastError() string {
|
||||
ptr := C.anydoc_last_error()
|
||||
if ptr == nil {
|
||||
return ""
|
||||
}
|
||||
defer C.anydoc_string_free(ptr)
|
||||
return cString(ptr)
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
package anydoc
|
||||
|
||||
/*
|
||||
#include "include/anydoc.h"
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Format names a document format. The string values are the lowercase
|
||||
// extension names, matching the Node and Python bindings ("doc", "docx",
|
||||
// "pdf", ...). Use the exported constants rather than raw strings where
|
||||
// possible: the constants are pinned to the C ABI's tag values at build
|
||||
// time.
|
||||
type Format string
|
||||
|
||||
// Format constants. Every value is stable across versions.
|
||||
const (
|
||||
FormatDoc Format = "doc"
|
||||
FormatDocx Format = "docx"
|
||||
FormatOdt Format = "odt"
|
||||
FormatPdf Format = "pdf"
|
||||
FormatPpt Format = "ppt"
|
||||
FormatPptx Format = "pptx"
|
||||
FormatRtf Format = "rtf"
|
||||
FormatEpub Format = "epub"
|
||||
FormatXlsx Format = "xlsx"
|
||||
FormatOds Format = "ods"
|
||||
FormatOdp Format = "odp"
|
||||
FormatCsv Format = "csv"
|
||||
)
|
||||
|
||||
// formatToTag maps a Format to its C ABI tag. Returns ANYDOC_FORMAT_NONE for
|
||||
// an unknown format.
|
||||
func formatToTag(f Format) C.int {
|
||||
switch f {
|
||||
case FormatDoc:
|
||||
return C.int(C.ANYDOC_FORMAT_DOC)
|
||||
case FormatDocx:
|
||||
return C.int(C.ANYDOC_FORMAT_DOCX)
|
||||
case FormatOdt:
|
||||
return C.int(C.ANYDOC_FORMAT_ODT)
|
||||
case FormatPdf:
|
||||
return C.int(C.ANYDOC_FORMAT_PDF)
|
||||
case FormatPpt:
|
||||
return C.int(C.ANYDOC_FORMAT_PPT)
|
||||
case FormatPptx:
|
||||
return C.int(C.ANYDOC_FORMAT_PPTX)
|
||||
case FormatRtf:
|
||||
return C.int(C.ANYDOC_FORMAT_RTF)
|
||||
case FormatEpub:
|
||||
return C.int(C.ANYDOC_FORMAT_EPUB)
|
||||
case FormatXlsx:
|
||||
return C.int(C.ANYDOC_FORMAT_XLSX)
|
||||
case FormatOds:
|
||||
return C.int(C.ANYDOC_FORMAT_ODS)
|
||||
case FormatOdp:
|
||||
return C.int(C.ANYDOC_FORMAT_ODP)
|
||||
case FormatCsv:
|
||||
return C.int(C.ANYDOC_FORMAT_CSV)
|
||||
}
|
||||
return C.int(C.ANYDOC_FORMAT_NONE)
|
||||
}
|
||||
|
||||
// tagToFormat maps a C ABI tag back to a Format. ok is false for NONE or an
|
||||
// unknown tag.
|
||||
func tagToFormat(tag C.int) (Format, bool) {
|
||||
switch tag {
|
||||
case C.ANYDOC_FORMAT_DOC:
|
||||
return FormatDoc, true
|
||||
case C.ANYDOC_FORMAT_DOCX:
|
||||
return FormatDocx, true
|
||||
case C.ANYDOC_FORMAT_ODT:
|
||||
return FormatOdt, true
|
||||
case C.ANYDOC_FORMAT_PDF:
|
||||
return FormatPdf, true
|
||||
case C.ANYDOC_FORMAT_PPT:
|
||||
return FormatPpt, true
|
||||
case C.ANYDOC_FORMAT_PPTX:
|
||||
return FormatPptx, true
|
||||
case C.ANYDOC_FORMAT_RTF:
|
||||
return FormatRtf, true
|
||||
case C.ANYDOC_FORMAT_EPUB:
|
||||
return FormatEpub, true
|
||||
case C.ANYDOC_FORMAT_XLSX:
|
||||
return FormatXlsx, true
|
||||
case C.ANYDOC_FORMAT_ODS:
|
||||
return FormatOds, true
|
||||
case C.ANYDOC_FORMAT_ODP:
|
||||
return FormatOdp, true
|
||||
case C.ANYDOC_FORMAT_CSV:
|
||||
return FormatCsv, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// FormatFromBytes detects the format from the content itself: the signature
|
||||
// and identity each container specification designates (PDF header, RTF open
|
||||
// group, OLE stream names, ZIP package mimetype/content types). Plain-text
|
||||
// formats (CSV) carry no signature and return the zero Format and ok=false;
|
||||
// so does anything unrecognized.
|
||||
func FormatFromBytes(data []byte) (f Format, ok bool) {
|
||||
if len(data) == 0 {
|
||||
return "", false
|
||||
}
|
||||
var tag C.int
|
||||
code := C.anydoc_format_from_bytes((*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), &tag)
|
||||
if code != errOK {
|
||||
return "", false
|
||||
}
|
||||
return tagToFormat(tag)
|
||||
}
|
||||
|
||||
// FormatFromExtension returns the format a bare extension names (with or
|
||||
// without a leading dot), matched case-insensitively. ok is false for
|
||||
// anything unrecognized.
|
||||
func FormatFromExtension(ext string) (f Format, ok bool) {
|
||||
ext = strings.TrimPrefix(ext, ".")
|
||||
cext := C.CString(strings.ToLower(ext))
|
||||
defer C.free(unsafe.Pointer(cext))
|
||||
var tag C.int
|
||||
code := C.anydoc_format_from_extension(cext, &tag)
|
||||
if code != errOK {
|
||||
return "", false
|
||||
}
|
||||
return tagToFormat(tag)
|
||||
}
|
||||
|
||||
// FormatFromPath returns the format a path's extension names. ok is false
|
||||
// when the path has no extension or names nothing recognized.
|
||||
func FormatFromPath(path string) (f Format, ok bool) {
|
||||
cpath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cpath))
|
||||
var tag C.int
|
||||
code := C.anydoc_format_from_path(cpath, &tag)
|
||||
if code != errOK {
|
||||
return "", false
|
||||
}
|
||||
return tagToFormat(tag)
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
module github.com/firecrawl/anydoc/go
|
||||
|
||||
go 1.22
|
||||
|
|
@ -0,0 +1,264 @@
|
|||
#ifndef ANYDOC_H
|
||||
#define ANYDOC_H
|
||||
|
||||
/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/**
|
||||
* Success.
|
||||
*/
|
||||
#define ERR_OK 0
|
||||
|
||||
/**
|
||||
* `ConvertError::Unsupported`.
|
||||
*/
|
||||
#define ERR_UNSUPPORTED 1
|
||||
|
||||
/**
|
||||
* `ConvertError::Malformed`.
|
||||
*/
|
||||
#define ERR_MALFORMED 2
|
||||
|
||||
/**
|
||||
* `ConvertError::Encrypted`.
|
||||
*/
|
||||
#define ERR_ENCRYPTED 3
|
||||
|
||||
/**
|
||||
* `ConvertError::ResourceLimit`.
|
||||
*/
|
||||
#define ERR_RESOURCE_LIMIT 4
|
||||
|
||||
/**
|
||||
* `ConvertError::MissingPart`.
|
||||
*/
|
||||
#define ERR_MISSING_PART 5
|
||||
|
||||
/**
|
||||
* `ConvertError::Io`.
|
||||
*/
|
||||
#define ERR_IO 6
|
||||
|
||||
/**
|
||||
* PDF was passed to `anydoc_to_document` (unsupported there).
|
||||
*/
|
||||
#define ERR_PDF_NO_MODEL 7
|
||||
|
||||
/**
|
||||
* Null pointer or invalid argument passed to the ABI.
|
||||
*/
|
||||
#define ERR_INVALID_ARG 8
|
||||
|
||||
/**
|
||||
* Unknown format name passed to the ABI.
|
||||
*/
|
||||
#define ERR_UNKNOWN_FORMAT 9
|
||||
|
||||
/**
|
||||
* C-side format tag. Stable; mirrors the Node/Python lowercase string names
|
||||
* via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel.
|
||||
*/
|
||||
#define ANYDOC_FORMAT_NONE -1
|
||||
|
||||
#define ANYDOC_FORMAT_DOC 0
|
||||
|
||||
#define ANYDOC_FORMAT_DOCX 1
|
||||
|
||||
#define ANYDOC_FORMAT_ODT 2
|
||||
|
||||
#define ANYDOC_FORMAT_PDF 3
|
||||
|
||||
#define ANYDOC_FORMAT_PPT 4
|
||||
|
||||
#define ANYDOC_FORMAT_PPTX 5
|
||||
|
||||
#define ANYDOC_FORMAT_RTF 6
|
||||
|
||||
#define ANYDOC_FORMAT_EPUB 7
|
||||
|
||||
#define ANYDOC_FORMAT_XLSX 8
|
||||
|
||||
#define ANYDOC_FORMAT_ODS 9
|
||||
|
||||
#define ANYDOC_FORMAT_ODP 10
|
||||
|
||||
#define ANYDOC_FORMAT_CSV 11
|
||||
|
||||
/**
|
||||
* Tag values for `Block` variants. Stable across versions; the Go side maps
|
||||
* these to the same lowercase string names the Node and Python bindings use.
|
||||
*/
|
||||
#define BLOCK_HEADING 0
|
||||
|
||||
#define BLOCK_PARAGRAPH 1
|
||||
|
||||
#define BLOCK_LIST 2
|
||||
|
||||
#define BLOCK_TABLE 3
|
||||
|
||||
#define BLOCK_QUOTE 4
|
||||
|
||||
#define BLOCK_CODE 5
|
||||
|
||||
#define BLOCK_RULE 6
|
||||
|
||||
#define INLINE_TEXT 0
|
||||
|
||||
#define INLINE_LINK 1
|
||||
|
||||
#define INLINE_IMAGE 2
|
||||
|
||||
#define INLINE_ANCHOR 3
|
||||
|
||||
#define INLINE_NOTEREF 4
|
||||
|
||||
#define INLINE_LINEBREAK 5
|
||||
|
||||
#define LINK_EXTERNAL 0
|
||||
|
||||
#define LINK_RELATIVE 1
|
||||
|
||||
#define LINK_ANCHOR 2
|
||||
|
||||
#define IMG_EXTERNAL 0
|
||||
|
||||
#define IMG_ASSET 1
|
||||
|
||||
#define IMG_UNAVAILABLE 2
|
||||
|
||||
#define MARKER_BULLET 0
|
||||
|
||||
#define MARKER_DECIMAL 1
|
||||
|
||||
#define MARKER_LOWER_ALPHA 2
|
||||
|
||||
#define MARKER_UPPER_ALPHA 3
|
||||
|
||||
#define MARKER_LOWER_ROMAN 4
|
||||
|
||||
#define MARKER_UPPER_ROMAN 5
|
||||
|
||||
#define TABLE_DATA 0
|
||||
|
||||
#define TABLE_LAYOUT 1
|
||||
|
||||
#define SLOT_ORIGIN 0
|
||||
|
||||
#define SLOT_COVERED 1
|
||||
|
||||
#define NOTE_FOOTNOTE 0
|
||||
|
||||
#define NOTE_ENDNOTE 1
|
||||
|
||||
/**
|
||||
* Detect the format from the content. Writes a format tag to `*out` (one of
|
||||
* `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches.
|
||||
*
|
||||
* Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null
|
||||
* pointers.
|
||||
*/
|
||||
int anydoc_format_from_bytes(const uint8_t *bytes, uintptr_t len, int *out);
|
||||
|
||||
/**
|
||||
* The format a bare extension names (no leading dot). Writes a format tag to
|
||||
* `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized.
|
||||
*/
|
||||
int anydoc_format_from_extension(const char *ext, int *out);
|
||||
|
||||
/**
|
||||
* The format a path's extension names. Writes a format tag to `*out`, or
|
||||
* `ANYDOC_FORMAT_NONE` when unrecognized.
|
||||
*/
|
||||
int anydoc_format_from_path(const char *path, int *out);
|
||||
|
||||
/**
|
||||
* Convert a document file to Markdown. The format is detected from the file
|
||||
* content, with the extension as the fallback for signature-less formats
|
||||
* (CSV).
|
||||
*
|
||||
* On success writes a `*mut c_char` to `*out_str` and its byte length to
|
||||
* `*out_len`. The caller must free the string with `anydoc_string_free`.
|
||||
*/
|
||||
int anydoc_to_markdown(const char *path, char **out_str, uintptr_t *out_len);
|
||||
|
||||
/**
|
||||
* Convert an in-memory document to Markdown. `format_tag` is one of the
|
||||
* `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the
|
||||
* content.
|
||||
*/
|
||||
int anydoc_to_markdown_bytes(const uint8_t *bytes,
|
||||
uintptr_t len,
|
||||
int format_tag,
|
||||
char **out_str,
|
||||
uintptr_t *out_len);
|
||||
|
||||
/**
|
||||
* Convert an in-memory document to Markdown with embedded images rewritten
|
||||
* as `` so they keep their original positions.
|
||||
*
|
||||
* This is `to_document` + anydoc's official GFM serializer. Asset images are
|
||||
* first turned into `ImageSource::External` URLs; the serializer already
|
||||
* emits those as ordinary Markdown image links. PDF has no document model
|
||||
* and is converted the same way as `anydoc_to_markdown_bytes`.
|
||||
*/
|
||||
int anydoc_to_markdown_with_asset_links(const uint8_t *bytes,
|
||||
uintptr_t len,
|
||||
int format_tag,
|
||||
char **out_str,
|
||||
uintptr_t *out_len);
|
||||
|
||||
/**
|
||||
* Parse an in-memory document into the document model. `format_tag` is one of
|
||||
* the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the
|
||||
* content.
|
||||
*
|
||||
* On success writes a heap-allocated byte buffer to `*out_buf` and its length
|
||||
* to `*out_len`. The buffer holds the flat serialization (see `model`). The
|
||||
* caller must free it with `anydoc_buffer_free`.
|
||||
*
|
||||
* Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown
|
||||
* directly and has no document-model form. Use `anydoc_to_markdown_bytes`.
|
||||
*/
|
||||
int anydoc_to_document(const uint8_t *bytes,
|
||||
uintptr_t len,
|
||||
int format_tag,
|
||||
uint8_t **out_buf,
|
||||
uintptr_t *out_len);
|
||||
|
||||
/**
|
||||
* Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`.
|
||||
* Null is a no-op (the empty-string sentinel).
|
||||
*/
|
||||
void anydoc_string_free(char *s);
|
||||
|
||||
/**
|
||||
* Free a document buffer returned by `anydoc_to_document`. Null is a no-op.
|
||||
* `len` must be the length written to `*out_len` at allocation time; it is
|
||||
* validated against the header for safety.
|
||||
*/
|
||||
void anydoc_buffer_free(uint8_t *buf, uintptr_t len);
|
||||
|
||||
/**
|
||||
* Return the human-readable message for the last error on the current
|
||||
* thread. The returned `*mut c_char` is freshly allocated and must be freed
|
||||
* with `anydoc_string_free`. Returns null when there is no last error.
|
||||
*/
|
||||
char *anydoc_last_error(void);
|
||||
|
||||
/**
|
||||
* Return the lowercase name of a format tag, or null for an unknown tag.
|
||||
* The returned string must be freed with `anydoc_string_free`.
|
||||
*/
|
||||
char *anydoc_format_name(int tag);
|
||||
|
||||
/**
|
||||
* Parse a lowercase format name into a tag. Writes the tag to `*out`, or
|
||||
* `ANYDOC_FORMAT_NONE` when the name is unknown.
|
||||
*/
|
||||
int anydoc_format_from_name(const char *name, int *out);
|
||||
|
||||
#endif /* ANYDOC_H */
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
//go:build linux && !musl
|
||||
|
||||
package anydoc
|
||||
|
||||
/*
|
||||
#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread
|
||||
#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread
|
||||
*/
|
||||
import "C"
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
//go:build linux && musl
|
||||
|
||||
package anydoc
|
||||
|
||||
/*
|
||||
#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread
|
||||
#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread
|
||||
*/
|
||||
import "C"
|
||||
|
|
@ -0,0 +1,729 @@
|
|||
package anydoc
|
||||
|
||||
/*
|
||||
#include "include/anydoc.h"
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Document is a parsed document: its body, its notes, and the bytes of
|
||||
// everything it embedded. A Document is self-contained — embedded assets
|
||||
// carry their bytes, so it stays usable after the source archive is gone.
|
||||
type Document struct {
|
||||
Blocks []Block
|
||||
// Notes are footnote and endnote bodies, referenced from text by an
|
||||
// Inline with Kind == "note_ref".
|
||||
Notes []Note
|
||||
// Assets holds every embedded asset, indexed by Asset.ID as referenced
|
||||
// by an Inline image source.
|
||||
Assets []Asset
|
||||
}
|
||||
|
||||
// Block is one block-level piece of a document body. The Kind field selects
|
||||
// which of the optional fields is populated; the rest are nil.
|
||||
type Block struct {
|
||||
// "heading", "paragraph", "list", "table", "block_quote", "code_block",
|
||||
// or "rule".
|
||||
Kind string
|
||||
Level *uint8 // heading: 1-based outline depth
|
||||
Anchor *string // heading: stable anchor id when the document targets it
|
||||
Content []Inline // heading, paragraph
|
||||
List *List // list
|
||||
Table *Table // table
|
||||
Blocks []Block // block_quote
|
||||
Lang *string // code_block
|
||||
Text *string // code_block
|
||||
}
|
||||
|
||||
// Inline is one span of inline content. The Kind field selects which optional
|
||||
// fields are populated.
|
||||
type Inline struct {
|
||||
// "text", "link", "image", "anchor", "note_ref", or "line_break".
|
||||
Kind string
|
||||
Text *string // text
|
||||
Style *Style // text
|
||||
Content []Inline // link
|
||||
Target *LinkTarget // link
|
||||
Alt *string // image
|
||||
Source *ImageSource // image
|
||||
Anchor *string // anchor: the anchor id
|
||||
NoteID *string // note_ref: the id of the note in Document.Notes
|
||||
}
|
||||
|
||||
// Style is a fully resolved character style.
|
||||
type Style struct {
|
||||
Bold bool
|
||||
Italic bool
|
||||
Strike bool
|
||||
Code bool
|
||||
}
|
||||
|
||||
// LinkTarget is where a link points.
|
||||
type LinkTarget struct {
|
||||
// "external" (absolute URL with a scheme), "relative" (scheme-less
|
||||
// relative reference, preserved as written), or "anchor" (internal
|
||||
// target: a heading anchor or an "anchor" inline).
|
||||
Kind string
|
||||
Value string // the URL, relative reference, or anchor id
|
||||
}
|
||||
|
||||
// ImageSource is where an image's bytes live.
|
||||
type ImageSource struct {
|
||||
// "external" (absolute URL), "asset" (embedded image, carried in
|
||||
// Document.Assets), or "unavailable" (no usable source; only the alt
|
||||
// text remains).
|
||||
Kind string
|
||||
URL *string // external
|
||||
AssetID *uint64 // asset: index into Document.Assets
|
||||
}
|
||||
|
||||
// List is a fully resolved list: numbering identity and marker resolution
|
||||
// happen in the parser frontends.
|
||||
type List struct {
|
||||
// "bullet", "decimal", "lower_alpha", "upper_alpha", "lower_roman", or
|
||||
// "upper_roman".
|
||||
Marker string
|
||||
Start uint64
|
||||
Items []ListItem
|
||||
}
|
||||
|
||||
// ListItem is one item of a List, which may hold nested blocks including
|
||||
// further lists.
|
||||
type ListItem struct {
|
||||
Blocks []Block
|
||||
Checked *bool // task-list state, when the item carries a checkbox
|
||||
MarkerLabel *string // literal marker text overriding the level marker
|
||||
}
|
||||
|
||||
// Table is a canonical table grid: every logical grid position appears
|
||||
// exactly once. Content and spans live on the origin slot, and each position
|
||||
// a span covers holds a CellSlot with Kind == "covered" pointing back at that
|
||||
// origin.
|
||||
type Table struct {
|
||||
Grid [][]CellSlot
|
||||
HeaderRows uint32 // number of leading rows that are header rows (0 = none)
|
||||
Kind string // "data" or "layout"
|
||||
}
|
||||
|
||||
// CellSlot is one position in a Table grid: either a cell or the shadow of
|
||||
// one.
|
||||
type CellSlot struct {
|
||||
// "origin" (the cell itself) or "covered" (a position swallowed by a
|
||||
// span, pointing back at the origin that covers it).
|
||||
Kind string
|
||||
Cell *Cell // origin
|
||||
OriginRow *uint32 // covered: row of the covering origin
|
||||
OriginCol *uint32 // covered: column of the covering origin
|
||||
}
|
||||
|
||||
// Cell is a table cell and the extent it spans.
|
||||
type Cell struct {
|
||||
Blocks []Block
|
||||
ColSpan uint32
|
||||
RowSpan uint32
|
||||
}
|
||||
|
||||
// Note is a footnote or endnote body, referenced from text by an Inline with
|
||||
// Kind == "note_ref".
|
||||
type Note struct {
|
||||
ID string
|
||||
Kind string // "footnote" or "endnote"
|
||||
Blocks []Block
|
||||
}
|
||||
|
||||
// Asset is an embedded binary asset (image, object payload). Bytes are always
|
||||
// retained, so a document stays self-contained.
|
||||
type Asset struct {
|
||||
ID uint64 // index into Document.Assets, as referenced by an image source
|
||||
MediaType string
|
||||
OriginPart string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// ---- Decoding the flat FFI buffer -----------------------------------------
|
||||
//
|
||||
// The buffer is the flat, length-prefixed serialization written by
|
||||
// go/src/model.rs. Every field is little-endian. A decoder tracks the read
|
||||
// cursor and reports a short or implausible buffer as an error — the Rust side
|
||||
// always produces a well-formed buffer, so a short read is a bug or a version
|
||||
// skew, not user input, and it must not take the process down.
|
||||
|
||||
type decoder struct {
|
||||
buf []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func decodeDocument(raw []byte) (*Document, error) {
|
||||
d := &decoder{buf: raw}
|
||||
doc := &Document{}
|
||||
var err error
|
||||
if doc.Blocks, err = d.blocks(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if doc.Notes, err = d.notes(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if doc.Assets, err = d.assets(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// capFor bounds a preallocation by what is left in the buffer. Every element
|
||||
// costs at least one byte to encode, so a count larger than the remaining
|
||||
// bytes can only come from a corrupt buffer or a Rust/Go version skew — and
|
||||
// preallocating it would exhaust memory before the first bounds check runs.
|
||||
func (d *decoder) capFor(n uint32) int {
|
||||
remaining := len(d.buf) - d.pos
|
||||
if remaining < 0 {
|
||||
return 0
|
||||
}
|
||||
if uint64(n) > uint64(remaining) {
|
||||
return remaining
|
||||
}
|
||||
return int(n)
|
||||
}
|
||||
|
||||
func (d *decoder) need(n int) error {
|
||||
// A negative n means a length prefix overflowed int (32-bit builds); it is
|
||||
// as unreadable as a truncated buffer.
|
||||
if n < 0 || d.pos+n > len(d.buf) {
|
||||
return fmt.Errorf("anydoc: truncated document buffer at offset %d (need %d bytes, have %d)", d.pos, n, len(d.buf)-d.pos)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *decoder) u8() (uint8, error) {
|
||||
if err := d.need(1); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := d.buf[d.pos]
|
||||
d.pos++
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (d *decoder) bool() (bool, error) {
|
||||
v, err := d.u8()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return v != 0, nil
|
||||
}
|
||||
|
||||
func (d *decoder) u32() (uint32, error) {
|
||||
if err := d.need(4); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := uint32(d.buf[d.pos]) | uint32(d.buf[d.pos+1])<<8 | uint32(d.buf[d.pos+2])<<16 | uint32(d.buf[d.pos+3])<<24
|
||||
d.pos += 4
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (d *decoder) u64() (uint64, error) {
|
||||
if err := d.need(8); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := uint64(d.buf[d.pos]) | uint64(d.buf[d.pos+1])<<8 | uint64(d.buf[d.pos+2])<<16 | uint64(d.buf[d.pos+3])<<24 |
|
||||
uint64(d.buf[d.pos+4])<<32 | uint64(d.buf[d.pos+5])<<40 | uint64(d.buf[d.pos+6])<<48 | uint64(d.buf[d.pos+7])<<56
|
||||
d.pos += 8
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (d *decoder) i32() (int32, error) {
|
||||
u, err := d.u32()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int32(u), nil
|
||||
}
|
||||
|
||||
// optStr reads a length-prefixed string or a -1 sentinel (None).
|
||||
func (d *decoder) optStr() (*string, error) {
|
||||
length, err := d.i32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length < 0 {
|
||||
return nil, nil
|
||||
}
|
||||
s, err := d.strN(int(length))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
func (d *decoder) str() (string, error) {
|
||||
length, err := d.u32()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.strN(int(length))
|
||||
}
|
||||
|
||||
func (d *decoder) strN(n int) (string, error) {
|
||||
if err := d.need(n); err != nil {
|
||||
return "", err
|
||||
}
|
||||
s := string(d.buf[d.pos : d.pos+n])
|
||||
d.pos += n
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (d *decoder) bytes() ([]byte, error) {
|
||||
length, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.need(int(length)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := make([]byte, length)
|
||||
copy(b, d.buf[d.pos:d.pos+int(length)])
|
||||
d.pos += int(length)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (d *decoder) blocks() ([]Block, error) {
|
||||
n, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Block, 0, d.capFor(n))
|
||||
for i := uint32(0); i < n; i++ {
|
||||
b, err := d.block()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *decoder) block() (Block, error) {
|
||||
tag, err := d.i32()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
switch tag {
|
||||
case C.BLOCK_HEADING:
|
||||
level, err := d.u8()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
anchor, err := d.optStr()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
content, err := d.inlines()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
lv := level
|
||||
return Block{Kind: "heading", Level: &lv, Anchor: anchor, Content: content}, nil
|
||||
case C.BLOCK_PARAGRAPH:
|
||||
content, err := d.inlines()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
return Block{Kind: "paragraph", Content: content}, nil
|
||||
case C.BLOCK_LIST:
|
||||
list, err := d.list()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
return Block{Kind: "list", List: list}, nil
|
||||
case C.BLOCK_TABLE:
|
||||
table, err := d.table()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
return Block{Kind: "table", Table: table}, nil
|
||||
case C.BLOCK_QUOTE:
|
||||
inner, err := d.blocks()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
return Block{Kind: "block_quote", Blocks: inner}, nil
|
||||
case C.BLOCK_CODE:
|
||||
lang, err := d.optStr()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
text, err := d.str()
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
return Block{Kind: "code_block", Lang: lang, Text: &text}, nil
|
||||
case C.BLOCK_RULE:
|
||||
return Block{Kind: "rule"}, nil
|
||||
default:
|
||||
return Block{}, fmt.Errorf("anydoc: unknown block kind tag %d", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) inlines() ([]Inline, error) {
|
||||
n, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Inline, 0, d.capFor(n))
|
||||
for i := uint32(0); i < n; i++ {
|
||||
in, err := d.inline()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, in)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *decoder) inline() (Inline, error) {
|
||||
tag, err := d.i32()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
switch tag {
|
||||
case C.INLINE_TEXT:
|
||||
text, err := d.str()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
style, err := d.style()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
return Inline{Kind: "text", Text: &text, Style: &style}, nil
|
||||
case C.INLINE_LINK:
|
||||
content, err := d.inlines()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
target, err := d.linkTarget()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
return Inline{Kind: "link", Content: content, Target: &target}, nil
|
||||
case C.INLINE_IMAGE:
|
||||
alt, err := d.str()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
src, err := d.imageSource()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
return Inline{Kind: "image", Alt: &alt, Source: &src}, nil
|
||||
case C.INLINE_ANCHOR:
|
||||
id, err := d.str()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
return Inline{Kind: "anchor", Anchor: &id}, nil
|
||||
case C.INLINE_NOTEREF:
|
||||
id, err := d.str()
|
||||
if err != nil {
|
||||
return Inline{}, err
|
||||
}
|
||||
return Inline{Kind: "note_ref", NoteID: &id}, nil
|
||||
case C.INLINE_LINEBREAK:
|
||||
return Inline{Kind: "line_break"}, nil
|
||||
default:
|
||||
return Inline{}, fmt.Errorf("anydoc: unknown inline kind tag %d", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) style() (Style, error) {
|
||||
bold, err := d.bool()
|
||||
if err != nil {
|
||||
return Style{}, err
|
||||
}
|
||||
italic, err := d.bool()
|
||||
if err != nil {
|
||||
return Style{}, err
|
||||
}
|
||||
strike, err := d.bool()
|
||||
if err != nil {
|
||||
return Style{}, err
|
||||
}
|
||||
code, err := d.bool()
|
||||
if err != nil {
|
||||
return Style{}, err
|
||||
}
|
||||
return Style{Bold: bold, Italic: italic, Strike: strike, Code: code}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) linkTarget() (LinkTarget, error) {
|
||||
tag, err := d.i32()
|
||||
if err != nil {
|
||||
return LinkTarget{}, err
|
||||
}
|
||||
value, err := d.str()
|
||||
if err != nil {
|
||||
return LinkTarget{}, err
|
||||
}
|
||||
switch tag {
|
||||
case C.LINK_EXTERNAL:
|
||||
return LinkTarget{Kind: "external", Value: value}, nil
|
||||
case C.LINK_RELATIVE:
|
||||
return LinkTarget{Kind: "relative", Value: value}, nil
|
||||
case C.LINK_ANCHOR:
|
||||
return LinkTarget{Kind: "anchor", Value: value}, nil
|
||||
default:
|
||||
return LinkTarget{}, fmt.Errorf("anydoc: unknown link target tag %d", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) imageSource() (ImageSource, error) {
|
||||
tag, err := d.i32()
|
||||
if err != nil {
|
||||
return ImageSource{}, err
|
||||
}
|
||||
switch tag {
|
||||
case C.IMG_EXTERNAL:
|
||||
url, err := d.str()
|
||||
if err != nil {
|
||||
return ImageSource{}, err
|
||||
}
|
||||
return ImageSource{Kind: "external", URL: &url}, nil
|
||||
case C.IMG_ASSET:
|
||||
id, err := d.u64()
|
||||
if err != nil {
|
||||
return ImageSource{}, err
|
||||
}
|
||||
return ImageSource{Kind: "asset", AssetID: &id}, nil
|
||||
case C.IMG_UNAVAILABLE:
|
||||
return ImageSource{Kind: "unavailable"}, nil
|
||||
default:
|
||||
return ImageSource{}, fmt.Errorf("anydoc: unknown image source tag %d", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) list() (*List, error) {
|
||||
markerTag, err := d.i32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, err := d.u64()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]ListItem, 0, d.capFor(n))
|
||||
for i := uint32(0); i < n; i++ {
|
||||
item, err := d.listItem()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
marker, err := markerName(markerTag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &List{Marker: marker, Start: start, Items: items}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) listItem() (ListItem, error) {
|
||||
blocks, err := d.blocks()
|
||||
if err != nil {
|
||||
return ListItem{}, err
|
||||
}
|
||||
checkedTag, err := d.i32()
|
||||
if err != nil {
|
||||
return ListItem{}, err
|
||||
}
|
||||
var checked *bool
|
||||
switch checkedTag {
|
||||
case 0:
|
||||
f := false
|
||||
checked = &f
|
||||
case 1:
|
||||
t := true
|
||||
checked = &t
|
||||
case -1:
|
||||
checked = nil
|
||||
default:
|
||||
return ListItem{}, fmt.Errorf("anydoc: unknown checked tag %d", checkedTag)
|
||||
}
|
||||
label, err := d.optStr()
|
||||
if err != nil {
|
||||
return ListItem{}, err
|
||||
}
|
||||
return ListItem{Blocks: blocks, Checked: checked, MarkerLabel: label}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) table() (*Table, error) {
|
||||
headerRows, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kindTag, err := d.i32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grid := make([][]CellSlot, 0, d.capFor(rows))
|
||||
for i := uint32(0); i < rows; i++ {
|
||||
cols, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row := make([]CellSlot, 0, d.capFor(cols))
|
||||
for j := uint32(0); j < cols; j++ {
|
||||
slot, err := d.slot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row = append(row, slot)
|
||||
}
|
||||
grid = append(grid, row)
|
||||
}
|
||||
kind := "data"
|
||||
if kindTag == C.TABLE_LAYOUT {
|
||||
kind = "layout"
|
||||
}
|
||||
return &Table{Grid: grid, HeaderRows: headerRows, Kind: kind}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) slot() (CellSlot, error) {
|
||||
tag, err := d.i32()
|
||||
if err != nil {
|
||||
return CellSlot{}, err
|
||||
}
|
||||
switch tag {
|
||||
case C.SLOT_ORIGIN:
|
||||
cell, err := d.cell()
|
||||
if err != nil {
|
||||
return CellSlot{}, err
|
||||
}
|
||||
return CellSlot{Kind: "origin", Cell: &cell}, nil
|
||||
case C.SLOT_COVERED:
|
||||
r, err := d.u32()
|
||||
if err != nil {
|
||||
return CellSlot{}, err
|
||||
}
|
||||
c, err := d.u32()
|
||||
if err != nil {
|
||||
return CellSlot{}, err
|
||||
}
|
||||
rr := r
|
||||
cc := c
|
||||
return CellSlot{Kind: "covered", OriginRow: &rr, OriginCol: &cc}, nil
|
||||
default:
|
||||
return CellSlot{}, fmt.Errorf("anydoc: unknown cell slot tag %d", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *decoder) cell() (Cell, error) {
|
||||
colSpan, err := d.u32()
|
||||
if err != nil {
|
||||
return Cell{}, err
|
||||
}
|
||||
rowSpan, err := d.u32()
|
||||
if err != nil {
|
||||
return Cell{}, err
|
||||
}
|
||||
blocks, err := d.blocks()
|
||||
if err != nil {
|
||||
return Cell{}, err
|
||||
}
|
||||
return Cell{Blocks: blocks, ColSpan: colSpan, RowSpan: rowSpan}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) notes() ([]Note, error) {
|
||||
n, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Note, 0, d.capFor(n))
|
||||
for i := uint32(0); i < n; i++ {
|
||||
note, err := d.note()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, note)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *decoder) note() (Note, error) {
|
||||
id, err := d.str()
|
||||
if err != nil {
|
||||
return Note{}, err
|
||||
}
|
||||
kindTag, err := d.i32()
|
||||
if err != nil {
|
||||
return Note{}, err
|
||||
}
|
||||
blocks, err := d.blocks()
|
||||
if err != nil {
|
||||
return Note{}, err
|
||||
}
|
||||
kind := "footnote"
|
||||
if kindTag == C.NOTE_ENDNOTE {
|
||||
kind = "endnote"
|
||||
}
|
||||
return Note{ID: id, Kind: kind, Blocks: blocks}, nil
|
||||
}
|
||||
|
||||
func (d *decoder) assets() ([]Asset, error) {
|
||||
n, err := d.u32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Asset, 0, d.capFor(n))
|
||||
for i := uint32(0); i < n; i++ {
|
||||
asset, err := d.asset()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, asset)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *decoder) asset() (Asset, error) {
|
||||
id, err := d.u64()
|
||||
if err != nil {
|
||||
return Asset{}, err
|
||||
}
|
||||
mediaType, err := d.str()
|
||||
if err != nil {
|
||||
return Asset{}, err
|
||||
}
|
||||
originPart, err := d.str()
|
||||
if err != nil {
|
||||
return Asset{}, err
|
||||
}
|
||||
data, err := d.bytes()
|
||||
if err != nil {
|
||||
return Asset{}, err
|
||||
}
|
||||
return Asset{ID: id, MediaType: mediaType, OriginPart: originPart, Data: data}, nil
|
||||
}
|
||||
|
||||
func markerName(tag int32) (string, error) {
|
||||
switch tag {
|
||||
case C.MARKER_BULLET:
|
||||
return "bullet", nil
|
||||
case C.MARKER_DECIMAL:
|
||||
return "decimal", nil
|
||||
case C.MARKER_LOWER_ALPHA:
|
||||
return "lower_alpha", nil
|
||||
case C.MARKER_UPPER_ALPHA:
|
||||
return "upper_alpha", nil
|
||||
case C.MARKER_LOWER_ROMAN:
|
||||
return "lower_roman", nil
|
||||
case C.MARKER_UPPER_ROMAN:
|
||||
return "upper_roman", nil
|
||||
}
|
||||
return "", fmt.Errorf("anydoc: unknown list marker tag %d", tag)
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
//! Rewrite embedded images to markdown-linkable sources so anydoc's official
|
||||
//! GFM serializer will emit `` in place.
|
||||
//!
|
||||
//! `document_to_markdown` drops `ImageSource::Asset` to alt text because
|
||||
//! Markdown cannot carry bytes. `ImageSource::External` is already rendered
|
||||
//! as a normal image link. We keep the official serializer and only change
|
||||
//! how those assets are addressed.
|
||||
|
||||
use anydoc::model::{Block, CellSlot, Document, ImageSource, Inline};
|
||||
|
||||
pub fn rewrite_asset_images(document: &mut Document) {
|
||||
let urls: Vec<Option<String>> = document
|
||||
.assets
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, asset)| {
|
||||
if asset.bytes.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(format!("images/image-{}{}", i + 1, extension_for(&asset.media_type)))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
rewrite_blocks(&mut document.blocks, &urls);
|
||||
for note in &mut document.notes {
|
||||
rewrite_blocks(&mut note.blocks, &urls);
|
||||
}
|
||||
}
|
||||
|
||||
fn rewrite_blocks(blocks: &mut [Block], urls: &[Option<String>]) {
|
||||
for block in blocks {
|
||||
match block {
|
||||
Block::Heading { content, .. } | Block::Paragraph(content) => {
|
||||
rewrite_inlines(content, urls);
|
||||
}
|
||||
Block::List(list) => {
|
||||
for item in &mut list.items {
|
||||
rewrite_blocks(&mut item.blocks, urls);
|
||||
}
|
||||
}
|
||||
Block::Table(table) => {
|
||||
for row in &mut table.grid {
|
||||
for slot in row {
|
||||
if let CellSlot::Origin(cell) = slot {
|
||||
rewrite_blocks(&mut cell.blocks, urls);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Block::BlockQuote(inner) => rewrite_blocks(inner, urls),
|
||||
Block::CodeBlock { .. } | Block::Rule => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rewrite_inlines(inlines: &mut [Inline], urls: &[Option<String>]) {
|
||||
for inline in inlines {
|
||||
match inline {
|
||||
Inline::Image { source, .. } => {
|
||||
if let ImageSource::Asset(id) = source {
|
||||
if let Some(Some(url)) = urls.get(id.0) {
|
||||
*source = ImageSource::External(url.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Inline::Link { content, .. } => rewrite_inlines(content, urls),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extension_for(media_type: &str) -> &'static str {
|
||||
match media_type.to_ascii_lowercase().as_str() {
|
||||
"image/jpeg" | "image/jpg" => ".jpg",
|
||||
"image/png" => ".png",
|
||||
"image/gif" => ".gif",
|
||||
"image/webp" => ".webp",
|
||||
"image/bmp" => ".bmp",
|
||||
"image/tiff" => ".tiff",
|
||||
"image/svg+xml" => ".svg",
|
||||
_ => ".bin",
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,620 @@
|
|||
//! C ABI surface for the Go bindings.
|
||||
//!
|
||||
//! Memory ownership rule: **Rust allocates, C ABI exposes a `_free` for every
|
||||
//! allocation family, Go releases handles.** There are two allocation
|
||||
//! families:
|
||||
//!
|
||||
//! - strings: `*mut c_char` + length out-params, returned by
|
||||
//! `anydoc_to_markdown*` and `anydoc_last_error`. Released via
|
||||
//! `anydoc_string_free`. Length is passed separately so binary-safe.
|
||||
//! - document buffer: a single `*mut u8` + length, returned by
|
||||
//! `anydoc_to_document`, holding the flat serialization from [`model`].
|
||||
//! Released via `anydoc_buffer_free`.
|
||||
//!
|
||||
//! Errors surface as stable non-zero codes (see `ERR_*`); the human-readable
|
||||
//! message is available through `anydoc_last_error`, thread-local.
|
||||
|
||||
#![allow(clippy::missing_safety_doc)]
|
||||
|
||||
use std::ffi::CString;
|
||||
use std::os::raw::{c_char, c_int};
|
||||
use std::ptr;
|
||||
|
||||
use anydoc::{ConvertError, Format, document_to_markdown};
|
||||
|
||||
mod asset_links;
|
||||
mod model;
|
||||
|
||||
// ---- Error codes -----------------------------------------------------------
|
||||
|
||||
/// Success.
|
||||
pub const ERR_OK: c_int = 0;
|
||||
/// `ConvertError::Unsupported`.
|
||||
pub const ERR_UNSUPPORTED: c_int = 1;
|
||||
/// `ConvertError::Malformed`.
|
||||
pub const ERR_MALFORMED: c_int = 2;
|
||||
/// `ConvertError::Encrypted`.
|
||||
pub const ERR_ENCRYPTED: c_int = 3;
|
||||
/// `ConvertError::ResourceLimit`.
|
||||
pub const ERR_RESOURCE_LIMIT: c_int = 4;
|
||||
/// `ConvertError::MissingPart`.
|
||||
pub const ERR_MISSING_PART: c_int = 5;
|
||||
/// `ConvertError::Io`.
|
||||
pub const ERR_IO: c_int = 6;
|
||||
/// PDF was passed to `anydoc_to_document` (unsupported there).
|
||||
pub const ERR_PDF_NO_MODEL: c_int = 7;
|
||||
/// Null pointer or invalid argument passed to the ABI.
|
||||
pub const ERR_INVALID_ARG: c_int = 8;
|
||||
/// Unknown format name passed to the ABI.
|
||||
pub const ERR_UNKNOWN_FORMAT: c_int = 9;
|
||||
|
||||
thread_local! {
|
||||
static LAST_ERROR: std::cell::RefCell<Option<CString>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn set_last_error(msg: &str) {
|
||||
let cstring = CString::new(msg).unwrap_or_else(|_| CString::new("<invalid utf-8>").unwrap());
|
||||
LAST_ERROR.with(|cell| *cell.borrow_mut() = Some(cstring));
|
||||
}
|
||||
|
||||
fn error_code(err: &ConvertError) -> c_int {
|
||||
match err {
|
||||
ConvertError::Unsupported(_) => ERR_UNSUPPORTED,
|
||||
ConvertError::Malformed { .. } => ERR_MALFORMED,
|
||||
ConvertError::Encrypted => ERR_ENCRYPTED,
|
||||
ConvertError::ResourceLimit { .. } => ERR_RESOURCE_LIMIT,
|
||||
ConvertError::MissingPart { .. } => ERR_MISSING_PART,
|
||||
ConvertError::Io(_) => ERR_IO,
|
||||
_ => ERR_UNSUPPORTED,
|
||||
}
|
||||
}
|
||||
|
||||
fn fail(err: ConvertError) -> c_int {
|
||||
let code = error_code(&err);
|
||||
set_last_error(&err.to_string());
|
||||
code
|
||||
}
|
||||
|
||||
/// Run a conversion with panics contained. A panic that escapes an
|
||||
/// `extern "C"` function aborts the process (Rust 1.81+), so one malformed
|
||||
/// document would take the whole host process down instead of failing a single
|
||||
/// conversion. A caught panic surfaces as `Malformed`, the same as any other
|
||||
/// unusable input.
|
||||
///
|
||||
/// This cannot contain an abort the runtime raises on its own (stack overflow,
|
||||
/// allocation failure); those still kill the process.
|
||||
fn guarded<T>(
|
||||
entry: &str,
|
||||
parse: impl FnOnce() -> Result<T, ConvertError>,
|
||||
) -> Result<T, ConvertError> {
|
||||
// AssertUnwindSafe: the closure only reads the input slice and returns an
|
||||
// owned result. Nothing observable is left half-written when it panics,
|
||||
// because the out-params are written by the caller after this returns.
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(parse)) {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(ConvertError::Malformed {
|
||||
part: None,
|
||||
detail: format!("panic while converting in {entry}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Format enum -----------------------------------------------------------
|
||||
|
||||
/// C-side format tag. Stable; mirrors the Node/Python lowercase string names
|
||||
/// via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel.
|
||||
pub const ANYDOC_FORMAT_NONE: c_int = -1;
|
||||
pub const ANYDOC_FORMAT_DOC: c_int = 0;
|
||||
pub const ANYDOC_FORMAT_DOCX: c_int = 1;
|
||||
pub const ANYDOC_FORMAT_ODT: c_int = 2;
|
||||
pub const ANYDOC_FORMAT_PDF: c_int = 3;
|
||||
pub const ANYDOC_FORMAT_PPT: c_int = 4;
|
||||
pub const ANYDOC_FORMAT_PPTX: c_int = 5;
|
||||
pub const ANYDOC_FORMAT_RTF: c_int = 6;
|
||||
pub const ANYDOC_FORMAT_EPUB: c_int = 7;
|
||||
pub const ANYDOC_FORMAT_XLSX: c_int = 8;
|
||||
pub const ANYDOC_FORMAT_ODS: c_int = 9;
|
||||
pub const ANYDOC_FORMAT_ODP: c_int = 10;
|
||||
pub const ANYDOC_FORMAT_CSV: c_int = 11;
|
||||
|
||||
fn format_to_c(format: Format) -> c_int {
|
||||
match format {
|
||||
Format::Doc => ANYDOC_FORMAT_DOC,
|
||||
Format::Docx => ANYDOC_FORMAT_DOCX,
|
||||
Format::Odt => ANYDOC_FORMAT_ODT,
|
||||
Format::Pdf => ANYDOC_FORMAT_PDF,
|
||||
Format::Ppt => ANYDOC_FORMAT_PPT,
|
||||
Format::Pptx => ANYDOC_FORMAT_PPTX,
|
||||
Format::Rtf => ANYDOC_FORMAT_RTF,
|
||||
Format::Epub => ANYDOC_FORMAT_EPUB,
|
||||
Format::Excel => ANYDOC_FORMAT_XLSX,
|
||||
Format::Ods => ANYDOC_FORMAT_ODS,
|
||||
Format::Odp => ANYDOC_FORMAT_ODP,
|
||||
Format::Csv => ANYDOC_FORMAT_CSV,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_from_c(tag: c_int) -> Option<Format> {
|
||||
Some(match tag {
|
||||
ANYDOC_FORMAT_DOC => Format::Doc,
|
||||
ANYDOC_FORMAT_DOCX => Format::Docx,
|
||||
ANYDOC_FORMAT_ODT => Format::Odt,
|
||||
ANYDOC_FORMAT_PDF => Format::Pdf,
|
||||
ANYDOC_FORMAT_PPT => Format::Ppt,
|
||||
ANYDOC_FORMAT_PPTX => Format::Pptx,
|
||||
ANYDOC_FORMAT_RTF => Format::Rtf,
|
||||
ANYDOC_FORMAT_EPUB => Format::Epub,
|
||||
ANYDOC_FORMAT_XLSX => Format::Excel,
|
||||
ANYDOC_FORMAT_ODS => Format::Ods,
|
||||
ANYDOC_FORMAT_ODP => Format::Odp,
|
||||
ANYDOC_FORMAT_CSV => Format::Csv,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Format names, lowercase, matching the Node/Python convention.
|
||||
const FORMAT_NAMES: [&str; 12] =
|
||||
["doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv"];
|
||||
|
||||
fn format_name(format: Format) -> &'static str {
|
||||
match format {
|
||||
Format::Doc => FORMAT_NAMES[0],
|
||||
Format::Docx => FORMAT_NAMES[1],
|
||||
Format::Odt => FORMAT_NAMES[2],
|
||||
Format::Pdf => FORMAT_NAMES[3],
|
||||
Format::Ppt => FORMAT_NAMES[4],
|
||||
Format::Pptx => FORMAT_NAMES[5],
|
||||
Format::Rtf => FORMAT_NAMES[6],
|
||||
Format::Epub => FORMAT_NAMES[7],
|
||||
Format::Excel => FORMAT_NAMES[8],
|
||||
Format::Ods => FORMAT_NAMES[9],
|
||||
Format::Odp => FORMAT_NAMES[10],
|
||||
Format::Csv => FORMAT_NAMES[11],
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_format_name(name: &str) -> Option<Format> {
|
||||
FORMAT_NAMES
|
||||
.iter()
|
||||
.zip([
|
||||
Format::Doc,
|
||||
Format::Docx,
|
||||
Format::Odt,
|
||||
Format::Pdf,
|
||||
Format::Ppt,
|
||||
Format::Pptx,
|
||||
Format::Rtf,
|
||||
Format::Epub,
|
||||
Format::Excel,
|
||||
Format::Ods,
|
||||
Format::Odp,
|
||||
Format::Csv,
|
||||
])
|
||||
.find(|(n, _)| **n == name)
|
||||
.map(|(_, f)| f)
|
||||
}
|
||||
|
||||
// ---- String out-param helper ----------------------------------------------
|
||||
|
||||
/// Box a `String` into a heap allocation the caller owns and must free with
|
||||
/// `anydoc_string_free`. Writes the byte length (excluding any NUL) to
|
||||
/// `*out_len`. The buffer is NUL-terminated for safety with C string readers,
|
||||
/// but binary-safe content relies on `out_len`.
|
||||
///
|
||||
/// Returns a dangling null pointer for empty strings so the Go side can treat
|
||||
/// a null result as "empty" without needing to free it. Length is still
|
||||
/// written as 0.
|
||||
fn string_out(s: String, out_str: *mut *mut c_char, out_len: *mut usize) -> c_int {
|
||||
if out_str.is_null() {
|
||||
set_last_error("anydoc: out_str is null");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let len = s.len();
|
||||
// SAFETY: we write the length unconditionally; null is the empty sentinel.
|
||||
if !out_len.is_null() {
|
||||
unsafe { *out_len = len };
|
||||
}
|
||||
if len == 0 {
|
||||
// Empty string: return null sentinel, no allocation to free.
|
||||
unsafe { *out_str = ptr::null_mut() };
|
||||
return ERR_OK;
|
||||
}
|
||||
let cstring = match CString::new(s.into_bytes()) {
|
||||
Ok(cs) => cs,
|
||||
Err(_) => {
|
||||
// Contains an interior NUL: fall back to lossy replacement so the
|
||||
// caller still gets something. Markdown shouldn't contain NULs.
|
||||
set_last_error("anydoc: output contained an interior NUL byte");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
};
|
||||
let raw = cstring.into_raw();
|
||||
// into_raw returns a buffer of capacity len+1 (with the trailing NUL).
|
||||
unsafe { *out_str = raw };
|
||||
ERR_OK
|
||||
}
|
||||
|
||||
// ---- C ABI functions -------------------------------------------------------
|
||||
|
||||
/// Detect the format from the content. Writes a format tag to `*out` (one of
|
||||
/// `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches.
|
||||
///
|
||||
/// Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null
|
||||
/// pointers.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_format_from_bytes(
|
||||
bytes: *const u8,
|
||||
len: usize,
|
||||
out: *mut c_int,
|
||||
) -> c_int {
|
||||
if bytes.is_null() || out.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_format_from_bytes");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
|
||||
let tag = Format::from_bytes(slice).map_or(ANYDOC_FORMAT_NONE, format_to_c);
|
||||
unsafe { *out = tag };
|
||||
ERR_OK
|
||||
}
|
||||
|
||||
/// The format a bare extension names (no leading dot). Writes a format tag to
|
||||
/// `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_format_from_extension(
|
||||
ext: *const c_char,
|
||||
out: *mut c_int,
|
||||
) -> c_int {
|
||||
if ext.is_null() || out.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_format_from_extension");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let ext_str = match unsafe { std::ffi::CStr::from_ptr(ext) }.to_str() {
|
||||
Ok(s) => s.trim_start_matches('.'),
|
||||
Err(_) => {
|
||||
set_last_error("anydoc: extension is not valid UTF-8");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
};
|
||||
let tag = Format::from_extension(ext_str).map_or(ANYDOC_FORMAT_NONE, format_to_c);
|
||||
unsafe { *out = tag };
|
||||
ERR_OK
|
||||
}
|
||||
|
||||
/// The format a path's extension names. Writes a format tag to `*out`, or
|
||||
/// `ANYDOC_FORMAT_NONE` when unrecognized.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_format_from_path(path: *const c_char, out: *mut c_int) -> c_int {
|
||||
if path.is_null() || out.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_format_from_path");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_last_error("anydoc: path is not valid UTF-8");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
};
|
||||
let tag =
|
||||
Format::from_path(std::path::Path::new(path_str)).map_or(ANYDOC_FORMAT_NONE, format_to_c);
|
||||
unsafe { *out = tag };
|
||||
ERR_OK
|
||||
}
|
||||
|
||||
/// Convert a document file to Markdown. The format is detected from the file
|
||||
/// content, with the extension as the fallback for signature-less formats
|
||||
/// (CSV).
|
||||
///
|
||||
/// On success writes a `*mut c_char` to `*out_str` and its byte length to
|
||||
/// `*out_len`. The caller must free the string with `anydoc_string_free`.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_to_markdown(
|
||||
path: *const c_char,
|
||||
out_str: *mut *mut c_char,
|
||||
out_len: *mut usize,
|
||||
) -> c_int {
|
||||
if path.is_null() || out_str.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_to_markdown");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_last_error("anydoc: path is not valid UTF-8");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
};
|
||||
match guarded("anydoc_to_markdown", || anydoc::to_markdown(path_str)) {
|
||||
Ok(md) => string_out(md, out_str, out_len),
|
||||
Err(e) => fail(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert an in-memory document to Markdown. `format_tag` is one of the
|
||||
/// `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the
|
||||
/// content.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_to_markdown_bytes(
|
||||
bytes: *const u8,
|
||||
len: usize,
|
||||
format_tag: c_int,
|
||||
out_str: *mut *mut c_char,
|
||||
out_len: *mut usize,
|
||||
) -> c_int {
|
||||
if bytes.is_null() || out_str.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_to_markdown_bytes");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let format = if format_tag == ANYDOC_FORMAT_NONE {
|
||||
None
|
||||
} else {
|
||||
match format_from_c(format_tag) {
|
||||
Some(f) => Some(f),
|
||||
None => {
|
||||
set_last_error("anydoc: unknown format tag");
|
||||
return ERR_UNKNOWN_FORMAT;
|
||||
}
|
||||
}
|
||||
};
|
||||
let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
|
||||
match guarded("anydoc_to_markdown_bytes", || anydoc::to_markdown_bytes(slice, format)) {
|
||||
Ok(md) => string_out(md, out_str, out_len),
|
||||
Err(e) => fail(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert an in-memory document to Markdown with embedded images rewritten
|
||||
/// as `` so they keep their original positions.
|
||||
///
|
||||
/// This is `to_document` + anydoc's official GFM serializer. Asset images are
|
||||
/// first turned into `ImageSource::External` URLs; the serializer already
|
||||
/// emits those as ordinary Markdown image links. PDF has no document model
|
||||
/// and is converted the same way as `anydoc_to_markdown_bytes`.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_to_markdown_with_asset_links(
|
||||
bytes: *const u8,
|
||||
len: usize,
|
||||
format_tag: c_int,
|
||||
out_str: *mut *mut c_char,
|
||||
out_len: *mut usize,
|
||||
) -> c_int {
|
||||
if bytes.is_null() || out_str.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_to_markdown_with_asset_links");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let format = if format_tag == ANYDOC_FORMAT_NONE {
|
||||
None
|
||||
} else {
|
||||
match format_from_c(format_tag) {
|
||||
Some(f) => Some(f),
|
||||
None => {
|
||||
set_last_error("anydoc: unknown format tag");
|
||||
return ERR_UNKNOWN_FORMAT;
|
||||
}
|
||||
}
|
||||
};
|
||||
let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
|
||||
match guarded("anydoc_to_markdown_with_asset_links", || {
|
||||
markdown_with_asset_links(slice, format)
|
||||
}) {
|
||||
Ok(md) => string_out(md, out_str, out_len),
|
||||
Err(e) => fail(e),
|
||||
}
|
||||
}
|
||||
|
||||
fn markdown_with_asset_links(
|
||||
bytes: &[u8],
|
||||
format: Option<Format>,
|
||||
) -> Result<String, ConvertError> {
|
||||
let resolved = format
|
||||
.or_else(|| Format::from_bytes(bytes))
|
||||
.ok_or_else(|| {
|
||||
ConvertError::Unsupported(
|
||||
"unrecognized file content: name the format explicitly".into(),
|
||||
)
|
||||
})?;
|
||||
if resolved == Format::Pdf {
|
||||
return anydoc::to_markdown_bytes(bytes, resolved);
|
||||
}
|
||||
let mut document = anydoc::to_document(bytes, resolved)?;
|
||||
asset_links::rewrite_asset_images(&mut document);
|
||||
Ok(document_to_markdown(&document))
|
||||
}
|
||||
|
||||
/// Parse an in-memory document into the document model. `format_tag` is one of
|
||||
/// the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the
|
||||
/// content.
|
||||
///
|
||||
/// On success writes a heap-allocated byte buffer to `*out_buf` and its length
|
||||
/// to `*out_len`. The buffer holds the flat serialization (see `model`). The
|
||||
/// caller must free it with `anydoc_buffer_free`.
|
||||
///
|
||||
/// Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown
|
||||
/// directly and has no document-model form. Use `anydoc_to_markdown_bytes`.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_to_document(
|
||||
bytes: *const u8,
|
||||
len: usize,
|
||||
format_tag: c_int,
|
||||
out_buf: *mut *mut u8,
|
||||
out_len: *mut usize,
|
||||
) -> c_int {
|
||||
if bytes.is_null() || out_buf.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_to_document");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let format = if format_tag == ANYDOC_FORMAT_NONE {
|
||||
None
|
||||
} else {
|
||||
match format_from_c(format_tag) {
|
||||
Some(f) => Some(f),
|
||||
None => {
|
||||
set_last_error("anydoc: unknown format tag");
|
||||
return ERR_UNKNOWN_FORMAT;
|
||||
}
|
||||
}
|
||||
};
|
||||
let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
|
||||
// Resolve the format up front so we can give the typed PDF error before
|
||||
// parsing, matching the crate's `to_document` contract.
|
||||
let resolved = match format {
|
||||
Some(f) => f,
|
||||
None => match Format::from_bytes(slice) {
|
||||
Some(f) => f,
|
||||
None => {
|
||||
set_last_error(
|
||||
"unsupported input: unrecognized file content: name the format explicitly",
|
||||
);
|
||||
return ERR_UNSUPPORTED;
|
||||
}
|
||||
},
|
||||
};
|
||||
if resolved == Format::Pdf {
|
||||
set_last_error(
|
||||
"unsupported input: PDF conversion produces Markdown directly and has no document-model form; use to_markdown_bytes",
|
||||
);
|
||||
return ERR_PDF_NO_MODEL;
|
||||
}
|
||||
let document = match guarded("anydoc_to_document", || anydoc::to_document(slice, resolved)) {
|
||||
Ok(d) => d,
|
||||
Err(e) => return fail(e),
|
||||
};
|
||||
let mut encoder = model::Encoder::new();
|
||||
model::write_document(&mut encoder, &document);
|
||||
let body = encoder.into_vec();
|
||||
let len = body.len();
|
||||
// Layout: [usize len][usize cap][u8 data..len]. The caller receives a
|
||||
// pointer to the data area and the length; `anydoc_buffer_free` walks back
|
||||
// to the header to recover (len, cap) for a sound deallocation.
|
||||
let header_size = std::mem::size_of::<BufferHeader>();
|
||||
let total = header_size + len;
|
||||
let layout = std::alloc::Layout::from_size_align(total, std::mem::align_of::<BufferHeader>())
|
||||
.expect("document buffer layout");
|
||||
let base = unsafe { std::alloc::alloc(layout) as *mut BufferHeader };
|
||||
if base.is_null() {
|
||||
set_last_error("anydoc: out of memory");
|
||||
return ERR_IO;
|
||||
}
|
||||
// The capacity we record is the *total* allocation size, so the free path
|
||||
// can reconstruct a layout of the same size to pass back to `dealloc`.
|
||||
unsafe {
|
||||
std::ptr::write(base, BufferHeader { len, cap: total });
|
||||
let data = (base as *mut u8).add(header_size);
|
||||
std::ptr::copy_nonoverlapping(body.as_ptr(), data, len);
|
||||
*out_buf = data;
|
||||
if !out_len.is_null() {
|
||||
*out_len = len;
|
||||
}
|
||||
}
|
||||
// `body` drops here, freeing its own backing allocation.
|
||||
ERR_OK
|
||||
}
|
||||
|
||||
/// Header prepended to every document buffer, so `anydoc_buffer_free` can
|
||||
/// soundly reclaim the allocation without any thread-local state. The data
|
||||
/// the caller sees starts immediately after this header.
|
||||
#[repr(C)]
|
||||
struct BufferHeader {
|
||||
len: usize,
|
||||
/// Total allocation size (header + data), used to rebuild the Layout for
|
||||
/// `dealloc`.
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
/// Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`.
|
||||
/// Null is a no-op (the empty-string sentinel).
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_string_free(s: *mut c_char) {
|
||||
if s.is_null() {
|
||||
return;
|
||||
}
|
||||
// SAFETY: the pointer was produced by `CString::into_raw`; reclaiming it
|
||||
// with `from_raw` is the matching deallocator.
|
||||
unsafe {
|
||||
let _ = CString::from_raw(s);
|
||||
}
|
||||
}
|
||||
|
||||
/// Free a document buffer returned by `anydoc_to_document`. Null is a no-op.
|
||||
/// `len` must be the length written to `*out_len` at allocation time; it is
|
||||
/// validated against the header for safety.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_buffer_free(buf: *mut u8, len: usize) {
|
||||
if buf.is_null() {
|
||||
return;
|
||||
}
|
||||
let header_size = std::mem::size_of::<BufferHeader>();
|
||||
// SAFETY: the pointer was produced by `anydoc_to_document` with a
|
||||
// `BufferHeader` immediately before it. Walk back to read the header and
|
||||
// rebuild the Layout, then dealloc the whole block.
|
||||
unsafe {
|
||||
let base = (buf.sub(header_size)) as *mut BufferHeader;
|
||||
let header = base.read();
|
||||
if header.len != len {
|
||||
// Length mismatch: caller bug. Leak rather than risk a bad free.
|
||||
set_last_error("anydoc: anydoc_buffer_free length does not match allocation");
|
||||
return;
|
||||
}
|
||||
let layout = std::alloc::Layout::from_size_align_unchecked(
|
||||
header.cap,
|
||||
std::mem::align_of::<BufferHeader>(),
|
||||
);
|
||||
std::alloc::dealloc(base as *mut u8, layout);
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the human-readable message for the last error on the current
|
||||
/// thread. The returned `*mut c_char` is freshly allocated and must be freed
|
||||
/// with `anydoc_string_free`. Returns null when there is no last error.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_last_error() -> *mut c_char {
|
||||
LAST_ERROR
|
||||
.with(|cell| cell.borrow_mut().take())
|
||||
.and_then(|cstring| {
|
||||
// Re-allocate as a fresh owned C string the caller can free.
|
||||
let bytes = cstring.into_bytes_with_nul();
|
||||
CString::from_vec_with_nul(bytes).ok().map(|cs| cs.into_raw())
|
||||
})
|
||||
.unwrap_or(ptr::null_mut())
|
||||
}
|
||||
|
||||
// ---- Go-side helpers exposed for parity tests -----------------------------
|
||||
|
||||
/// Return the lowercase name of a format tag, or null for an unknown tag.
|
||||
/// The returned string must be freed with `anydoc_string_free`.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_format_name(tag: c_int) -> *mut c_char {
|
||||
if tag == ANYDOC_FORMAT_NONE {
|
||||
return ptr::null_mut();
|
||||
}
|
||||
match format_from_c(tag) {
|
||||
Some(f) => {
|
||||
let name = format_name(f);
|
||||
match CString::new(name) {
|
||||
Ok(cs) => cs.into_raw(),
|
||||
Err(_) => ptr::null_mut(),
|
||||
}
|
||||
}
|
||||
None => ptr::null_mut(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a lowercase format name into a tag. Writes the tag to `*out`, or
|
||||
/// `ANYDOC_FORMAT_NONE` when the name is unknown.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn anydoc_format_from_name(name: *const c_char, out: *mut c_int) -> c_int {
|
||||
if name.is_null() || out.is_null() {
|
||||
set_last_error("anydoc: null pointer passed to anydoc_format_from_name");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
let name_str = match unsafe { std::ffi::CStr::from_ptr(name) }.to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_last_error("anydoc: format name is not valid UTF-8");
|
||||
return ERR_INVALID_ARG;
|
||||
}
|
||||
};
|
||||
let tag = parse_format_name(name_str).map_or(ANYDOC_FORMAT_NONE, format_to_c);
|
||||
unsafe { *out = tag };
|
||||
ERR_OK
|
||||
}
|
||||
|
|
@ -0,0 +1,321 @@
|
|||
//! `#[repr(C)]` DTOs and a flat, self-describing serialization of the document
|
||||
//! model.
|
||||
//!
|
||||
//! The document tree is recursive (blocks hold blocks, inlines hold inlines,
|
||||
//! tables hold cells that hold blocks), so rather than surfacing it through a
|
||||
//! forest of opaque handles and one `_free` per node type, the FFI serializes
|
||||
//! the whole `Document` into a single length-prefixed byte buffer. That gives
|
||||
//! exactly one allocation family for the tree (`anydoc_buffer_free`) and keeps
|
||||
//! the C ABI tiny. Variant tags are written inline as `c_int`-compatible
|
||||
//! values, mirroring the Node/Python `kind` string convention.
|
||||
//!
|
||||
//! Format: see `write_document` below. Every field is length-prefixed so the
|
||||
//! Go side can decode without guessing. Strings carry their byte length as a
|
||||
//! u32; slices carry a u32 count followed by that many elements. Booleans are
|
||||
//! a single byte (0 or 1). Integers are little-endian fixed width.
|
||||
|
||||
use std::os::raw::c_int;
|
||||
|
||||
use anydoc::model;
|
||||
|
||||
/// Tag values for `Block` variants. Stable across versions; the Go side maps
|
||||
/// these to the same lowercase string names the Node and Python bindings use.
|
||||
pub const BLOCK_HEADING: c_int = 0;
|
||||
pub const BLOCK_PARAGRAPH: c_int = 1;
|
||||
pub const BLOCK_LIST: c_int = 2;
|
||||
pub const BLOCK_TABLE: c_int = 3;
|
||||
pub const BLOCK_QUOTE: c_int = 4;
|
||||
pub const BLOCK_CODE: c_int = 5;
|
||||
pub const BLOCK_RULE: c_int = 6;
|
||||
|
||||
pub const INLINE_TEXT: c_int = 0;
|
||||
pub const INLINE_LINK: c_int = 1;
|
||||
pub const INLINE_IMAGE: c_int = 2;
|
||||
pub const INLINE_ANCHOR: c_int = 3;
|
||||
pub const INLINE_NOTEREF: c_int = 4;
|
||||
pub const INLINE_LINEBREAK: c_int = 5;
|
||||
|
||||
pub const LINK_EXTERNAL: c_int = 0;
|
||||
pub const LINK_RELATIVE: c_int = 1;
|
||||
pub const LINK_ANCHOR: c_int = 2;
|
||||
|
||||
pub const IMG_EXTERNAL: c_int = 0;
|
||||
pub const IMG_ASSET: c_int = 1;
|
||||
pub const IMG_UNAVAILABLE: c_int = 2;
|
||||
|
||||
pub const MARKER_BULLET: c_int = 0;
|
||||
pub const MARKER_DECIMAL: c_int = 1;
|
||||
pub const MARKER_LOWER_ALPHA: c_int = 2;
|
||||
pub const MARKER_UPPER_ALPHA: c_int = 3;
|
||||
pub const MARKER_LOWER_ROMAN: c_int = 4;
|
||||
pub const MARKER_UPPER_ROMAN: c_int = 5;
|
||||
|
||||
pub const TABLE_DATA: c_int = 0;
|
||||
pub const TABLE_LAYOUT: c_int = 1;
|
||||
|
||||
pub const SLOT_ORIGIN: c_int = 0;
|
||||
pub const SLOT_COVERED: c_int = 1;
|
||||
|
||||
pub const NOTE_FOOTNOTE: c_int = 0;
|
||||
pub const NOTE_ENDNOTE: c_int = 1;
|
||||
|
||||
/// Encoder for the flat document buffer. Little-endian, length-prefixed.
|
||||
pub struct Encoder {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Encoder {
|
||||
pub fn new() -> Self {
|
||||
Encoder { buf: Vec::with_capacity(8 * 1024) }
|
||||
}
|
||||
|
||||
pub fn into_vec(self) -> Vec<u8> {
|
||||
self.buf
|
||||
}
|
||||
|
||||
pub fn u8(&mut self, v: u8) {
|
||||
self.buf.push(v);
|
||||
}
|
||||
|
||||
pub fn bool(&mut self, v: bool) {
|
||||
self.buf.push(if v { 1 } else { 0 });
|
||||
}
|
||||
|
||||
pub fn u32(&mut self, v: u32) {
|
||||
self.buf.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
pub fn u64(&mut self, v: u64) {
|
||||
self.buf.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
pub fn i32(&mut self, v: i32) {
|
||||
self.buf.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// A length-prefixed UTF-8 string. `None` writes a -1 length sentinel.
|
||||
pub fn opt_str(&mut self, v: &Option<String>) {
|
||||
match v {
|
||||
Some(s) => self.str(s),
|
||||
None => self.i32(-1),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn str(&mut self, s: &str) {
|
||||
self.u32(s.len() as u32);
|
||||
self.buf.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
/// A length-prefixed byte slice.
|
||||
pub fn bytes(&mut self, b: &[u8]) {
|
||||
self.u32(b.len() as u32);
|
||||
self.buf.extend_from_slice(b);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn write_document(e: &mut Encoder, doc: &model::Document) {
|
||||
write_blocks(e, &doc.blocks);
|
||||
e.u32(doc.notes.len() as u32);
|
||||
for note in &doc.notes {
|
||||
write_note(e, note);
|
||||
}
|
||||
e.u32(doc.assets.len() as u32);
|
||||
for asset in &doc.assets {
|
||||
write_asset(e, asset);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_blocks(e: &mut Encoder, blocks: &[model::Block]) {
|
||||
e.u32(blocks.len() as u32);
|
||||
for block in blocks {
|
||||
write_block(e, block);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_block(e: &mut Encoder, block: &model::Block) {
|
||||
match block {
|
||||
model::Block::Heading { level, anchor, content } => {
|
||||
e.i32(BLOCK_HEADING);
|
||||
e.u8(*level);
|
||||
e.opt_str(anchor);
|
||||
write_inlines(e, content);
|
||||
}
|
||||
model::Block::Paragraph(content) => {
|
||||
e.i32(BLOCK_PARAGRAPH);
|
||||
write_inlines(e, content);
|
||||
}
|
||||
model::Block::List(list) => {
|
||||
e.i32(BLOCK_LIST);
|
||||
write_list(e, list);
|
||||
}
|
||||
model::Block::Table(table) => {
|
||||
e.i32(BLOCK_TABLE);
|
||||
write_table(e, table);
|
||||
}
|
||||
model::Block::BlockQuote(inner) => {
|
||||
e.i32(BLOCK_QUOTE);
|
||||
write_blocks(e, inner);
|
||||
}
|
||||
model::Block::CodeBlock { lang, text } => {
|
||||
e.i32(BLOCK_CODE);
|
||||
e.opt_str(lang);
|
||||
e.str(text);
|
||||
}
|
||||
model::Block::Rule => {
|
||||
e.i32(BLOCK_RULE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_inlines(e: &mut Encoder, inlines: &[model::Inline]) {
|
||||
e.u32(inlines.len() as u32);
|
||||
for inline in inlines {
|
||||
write_inline(e, inline);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_inline(e: &mut Encoder, inline: &model::Inline) {
|
||||
match inline {
|
||||
model::Inline::Text { text, style } => {
|
||||
e.i32(INLINE_TEXT);
|
||||
e.str(text);
|
||||
write_style(e, *style);
|
||||
}
|
||||
model::Inline::Link { content, target } => {
|
||||
e.i32(INLINE_LINK);
|
||||
write_inlines(e, content);
|
||||
write_link_target(e, target);
|
||||
}
|
||||
model::Inline::Image { alt, source } => {
|
||||
e.i32(INLINE_IMAGE);
|
||||
e.str(alt);
|
||||
write_image_source(e, source);
|
||||
}
|
||||
model::Inline::Anchor(id) => {
|
||||
e.i32(INLINE_ANCHOR);
|
||||
e.str(id);
|
||||
}
|
||||
model::Inline::NoteRef(id) => {
|
||||
e.i32(INLINE_NOTEREF);
|
||||
e.str(id);
|
||||
}
|
||||
model::Inline::LineBreak => {
|
||||
e.i32(INLINE_LINEBREAK);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_style(e: &mut Encoder, style: model::Style) {
|
||||
e.bool(style.bold);
|
||||
e.bool(style.italic);
|
||||
e.bool(style.strike);
|
||||
e.bool(style.code);
|
||||
}
|
||||
|
||||
fn write_link_target(e: &mut Encoder, target: &model::LinkTarget) {
|
||||
match target {
|
||||
model::LinkTarget::External(v) => {
|
||||
e.i32(LINK_EXTERNAL);
|
||||
e.str(v);
|
||||
}
|
||||
model::LinkTarget::Relative(v) => {
|
||||
e.i32(LINK_RELATIVE);
|
||||
e.str(v);
|
||||
}
|
||||
model::LinkTarget::Anchor(v) => {
|
||||
e.i32(LINK_ANCHOR);
|
||||
e.str(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_image_source(e: &mut Encoder, source: &model::ImageSource) {
|
||||
match source {
|
||||
model::ImageSource::External(url) => {
|
||||
e.i32(IMG_EXTERNAL);
|
||||
e.str(url);
|
||||
}
|
||||
model::ImageSource::Asset(id) => {
|
||||
e.i32(IMG_ASSET);
|
||||
e.u64(id.0 as u64);
|
||||
}
|
||||
model::ImageSource::Unavailable => {
|
||||
e.i32(IMG_UNAVAILABLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_list(e: &mut Encoder, list: &model::List) {
|
||||
e.i32(marker_tag(list.marker));
|
||||
e.u64(list.start);
|
||||
e.u32(list.items.len() as u32);
|
||||
for item in &list.items {
|
||||
write_list_item(e, item);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_list_item(e: &mut Encoder, item: &model::ListItem) {
|
||||
write_blocks(e, &item.blocks);
|
||||
match item.checked {
|
||||
None => e.i32(-1),
|
||||
Some(false) => e.i32(0),
|
||||
Some(true) => e.i32(1),
|
||||
}
|
||||
e.opt_str(&item.marker_label);
|
||||
}
|
||||
|
||||
fn write_table(e: &mut Encoder, table: &model::Table) {
|
||||
e.u32(table.header_rows as u32);
|
||||
e.i32(if table.kind == model::TableKind::Layout { TABLE_LAYOUT } else { TABLE_DATA });
|
||||
e.u32(table.grid.len() as u32);
|
||||
for row in &table.grid {
|
||||
e.u32(row.len() as u32);
|
||||
for slot in row {
|
||||
write_slot(e, slot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_slot(e: &mut Encoder, slot: &model::CellSlot) {
|
||||
match slot {
|
||||
model::CellSlot::Origin(cell) => {
|
||||
e.i32(SLOT_ORIGIN);
|
||||
write_cell(e, cell);
|
||||
}
|
||||
model::CellSlot::Covered { origin_row, origin_col } => {
|
||||
e.i32(SLOT_COVERED);
|
||||
e.u32(*origin_row as u32);
|
||||
e.u32(*origin_col as u32);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_cell(e: &mut Encoder, cell: &model::Cell) {
|
||||
e.u32(cell.col_span);
|
||||
e.u32(cell.row_span);
|
||||
write_blocks(e, &cell.blocks);
|
||||
}
|
||||
|
||||
fn write_note(e: &mut Encoder, note: &model::Note) {
|
||||
e.str(¬e.id);
|
||||
e.i32(if note.kind == model::NoteKind::Endnote { NOTE_ENDNOTE } else { NOTE_FOOTNOTE });
|
||||
write_blocks(e, ¬e.blocks);
|
||||
}
|
||||
|
||||
fn write_asset(e: &mut Encoder, asset: &model::Asset) {
|
||||
e.u64(asset.id.0 as u64);
|
||||
e.str(&asset.media_type);
|
||||
e.str(&asset.origin_part);
|
||||
e.bytes(&asset.bytes);
|
||||
}
|
||||
|
||||
fn marker_tag(marker: model::MarkerKind) -> c_int {
|
||||
match marker {
|
||||
model::MarkerKind::Bullet => MARKER_BULLET,
|
||||
model::MarkerKind::Decimal => MARKER_DECIMAL,
|
||||
model::MarkerKind::LowerAlpha => MARKER_LOWER_ALPHA,
|
||||
model::MarkerKind::UpperAlpha => MARKER_UPPER_ALPHA,
|
||||
model::MarkerKind::LowerRoman => MARKER_LOWER_ROMAN,
|
||||
model::MarkerKind::UpperRoman => MARKER_UPPER_ROMAN,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package anydoc
|
||||
|
||||
// Version is the version of the anydoc-go module, bumped in lockstep with the
|
||||
// Rust crate version (go/Cargo.toml) and the GitHub release tag. The
|
||||
// packaged static libraries.
|
||||
const Version = "0.1.8"
|
||||
|
|
@ -282,7 +282,10 @@ Worker 消费 `TypeDocumentProcess` 后按五个规范化阶段推进,每个
|
|||
4. `resolveDocReader` 返回 `interfaces.DocReader`:
|
||||
- **builtin**:通过 gRPC(`docparser/grpc_parser.go`)或 HTTP(`http_parser.go`)调用 Python **docreader** 服务;
|
||||
- **simple**:Go 原生解析 md/txt/csv/json/图片/音频(`builtin_converter.go`,CSV→Markdown 表格、JSON→递归分割的代码块,图片/音频转占位引用);
|
||||
- **weknoracloud / mineru / mineru_cloud / paddleocr_vl / paddleocr_vl_cloud**:HTTP 转换器(`engine_registry.go` 注册,按 `mineru_endpoint`、`mineru_api_key`、`paddleocr_vl_endpoint` 等配置判定可用性)。
|
||||
- **anydoc**:Go 进程内解析 docx/doc/pptx/ppt/xlsx/xls/odf/rtf/epub/csv/pdf(`anydoc_reader.go`),底层是通过 cgo 链接的 anydoc Rust 库。office 文档的嵌入图按文档模型插回 Markdown 原位;无文字层的扫描件 PDF 在 DocReader 可用时回退到 builtin 整页渲染。仅在带 `anydoc` 构建标签的二进制中可用,其余构建里该引擎在引擎列表中显示为不可用;
|
||||
- **weknoracloud / mineru / mineru_cloud / paddleocr_vl / paddleocr_vl_cloud**:HTTP 转换器(`engines.go` 注册,按 `mineru_endpoint`、`mineru_api_key`、`paddleocr_vl_endpoint` 等配置判定可用性)。
|
||||
|
||||
引擎目录集中在 `internal/infrastructure/docparser/engines.go`:每个引擎同时声明元数据(名称、描述、文件类型、可用性探针)与 `NewReader` 工厂,`docparser.NewReader` 按名字分发,未注册的名字(如只存在于 docreader 的 `markitdown`)落到 docreader 客户端。
|
||||
5. 文件模式:从 `FileService.GetFile(payload.FilePath)` 读回字节填入 `ReadRequest.FileContent`。
|
||||
|
||||
**docreader 服务侧**(`docreader/`,Python gRPC):proto 定义 `docreader/proto/docreader.proto`,服务方法 `Read` / `ReadStream`(流式:首帧 meta + 每图一帧,避免大扫描件 PDF 触发 gRPC 消息上限)/ `ListEngines`。内置 parser 覆盖 docx/doc/pdf/md/xlsx/xls/epub/html/htm/mhtml/图片/网页(`WebParser` 处理 URL),并可选注册 `markitdown`(微软 MarkItDown)与 `opendataloader`(PDF 版面分析,需 Java 11+)引擎,Go 侧通过 `ListEngines` 自发现远程引擎。返回统一为 `ReadResult{MarkdownContent, ImageRefs, Metadata, IsAudio, AudioData}` —— **解析产物统一是 Markdown 文本 + 图片字节**,图片持久化由 Go 侧负责。
|
||||
|
|
|
|||
|
|
@ -400,6 +400,48 @@ Python 依赖(`pyproject.toml` + `uv.lock` 锁定):`grpcio`、`pypdfium2`
|
|||
|
||||
---
|
||||
|
||||
## 8. anydoc 引擎(Go 进程内解析,不经 docreader)
|
||||
|
||||
`anydoc` 是一个 Go 侧的可选解析引擎:它把 [anydoc](https://github.com/firecrawl/anydoc)(Rust 编写的文档转换库)通过 cgo 链接进 WeKnora 主进程,直接把 office 文档转成 Markdown。与本文其余部分描述的 docreader 不同,它**不经过 Python 服务、不跨进程、也不调用外部二进制**——适合不想部署 docreader 的轻量部署,或对解析延迟敏感的场景。
|
||||
|
||||
支持的文件类型:`doc`、`docx`、`docm`、`odt`、`rtf`、`ppt`、`pptx`、`pptm`、`odp`、`xls`、`xlsx`、`xlsm`、`ods`、`epub`、`csv`、`pdf`。
|
||||
|
||||
### 8.1 启用方式
|
||||
|
||||
解析库是 Rust 静态库,需要 Rust 工具链构建,因此默认不链接:未启用时该引擎在「解析引擎」列表里显示为不可用,并给出提示,其它引擎不受影响。
|
||||
|
||||
```bash
|
||||
make build-anydoc # 构建静态库 + 带 anydoc 标签的二进制
|
||||
# 等价于:
|
||||
scripts/build-anydoc-lib.sh && go build -tags anydoc ./cmd/server
|
||||
```
|
||||
|
||||
Docker 镜像:
|
||||
|
||||
```bash
|
||||
docker build -f docker/Dockerfile.app --build-arg WITH_ANYDOC=1 -t weknora-app:anydoc .
|
||||
```
|
||||
|
||||
启用后在知识库的解析设置里把对应文件类型指向 `anydoc` 引擎即可。
|
||||
|
||||
### 8.2 能力边界
|
||||
|
||||
- **扫描件 PDF**:anydoc 只抽取 PDF 的文字层。没有文字层的扫描件会报「需要 OCR」;若 DocReader(builtin)已连接,AnydocReader 会自动把该文件交给 builtin,按页渲染 JPEG 并标记 `image_source_type=scanned_pdf`,后续仍走 Go 侧 OCR。未连接 DocReader 时转换失败,请改用 `builtin`、`mineru` 或 `paddleocr_vl`。
|
||||
- **纵向合并单元格不回填**:docreader 的 `Docx2Parser` 会把纵向合并的值复制到每一行(见 issue #2634),anydoc 只在起始行输出该值,后续行留空。对依赖表格逐行语义的知识库,`builtin` 仍然更稳。
|
||||
- **图片位置**:开启图片抽取时,先把文档模型里的嵌入图改写成 `images/image-N.ext` 链接,再交给 anydoc 官方 GFM 序列化,因此图片会留在原段落/表格/列表位置。设置引擎覆盖参数 `anydoc_extract_images=false` 可关闭图片抽取,走更快的纯文本渲染(嵌入图会退化成 alt 文本)。
|
||||
- **不处理 URL、图片、音频**:这些仍由 `WebParser`、`SimpleFormatReader` 与 ASR 链路负责。
|
||||
|
||||
### 8.3 代码位置
|
||||
|
||||
| 路径 | 作用 |
|
||||
| --- | --- |
|
||||
| `internal/infrastructure/docparser/anydoc/` | 适配层:格式映射、可用性判断,以及 cgo / stub 两个后端 |
|
||||
| `internal/infrastructure/docparser/anydoc_reader.go` | `DocReader` 实现:转换结果与图片引用组装 |
|
||||
| `internal/infrastructure/docparser/engines.go` | 引擎注册(元数据 + Reader 工厂) |
|
||||
| `third_party/anydoc-go/` | vendored 的上游 Go 绑定与 C ABI shim(来源与本地改动见该目录 README) |
|
||||
|
||||
---
|
||||
|
||||
## 附:关键事实速查
|
||||
|
||||
- **对外接口**:仅 gRPC,端口 `50051`(`DOCREADER_GRPC_PORT`/`PORT`),RPC:`Read` / `ReadStream` / `ListEngines` + 标准 Health 服务。
|
||||
|
|
|
|||
Loading…
Reference in New Issue