refactor(image): turn the hard-coded image flow into configurable pipelines (#3746)
* feat(image): configurable image pipelines, rebuilt on current main Rebuilds the image-pipeline work on top of the current main, carrying only the pipeline and request-side changes: - The hard-coded image flow becomes registry-driven pipelines (default / smartocr), selectable per knowledge base, with a frontend panel and per-action thinking switches addressed as action-level keys (caption.thinking / ocr.thinking) declared once and carried into every pipeline. - Request-side support: VLM.PredictWithOptions carries a per-call thinking switch through every implementation and decorator, including the background concurrency wrapper. - OCR failure handling follows main's #4132 semantics: invalid output is rejected and recorded (ocr_status / ocr_error_code / .ocr subspan) without retrying the whole image; caption/observation failures get symmetric subspans instead of leaving the image span green. - Permanent VLM failures skip the asynq retry budget; a run that produced nothing fails honestly instead of passing as skipped. - UI and locale fixes travel with the panel. Concurrency, rate limiting and circuit breaking are NOT part of this branch: they stay with the existing system limiter. * fix(image): address pipeline configuration and execution regressions * fix(image): align pipeline overrides and validation semantics * style(image): keep the new pipeline code inside the lll and gofumpt gates --------- Co-authored-by: tayadwx <tayadwx@users.noreply.github.com> Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
2fbfb44715
commit
db7046b85e
|
|
@ -220,12 +220,83 @@ export interface ImageProcessingConfig {
|
|||
model_id?: string;
|
||||
image_actions?: ImageActionsConfig;
|
||||
image_attrs_enabled?: boolean;
|
||||
/** Which image pipeline runs for this base; empty means "follow the switch". */
|
||||
image_pipeline?: string;
|
||||
/** That pipeline's private tunables, keyed by field. See ImagePipelineField. */
|
||||
image_pipeline_params?: Record<string, unknown>;
|
||||
// Index and recall images by their own vectors. Off unless the knowledge
|
||||
// base opts in, and only takes effect with an embedding model that takes
|
||||
// images.
|
||||
image_vector_enabled?: boolean;
|
||||
}
|
||||
|
||||
// Mirrors types.ImagePipelineSpec and types.ImageFieldDef.
|
||||
//
|
||||
// A field is private to the pipeline that declared it, which is why two
|
||||
// pipelines may both own a field called "enable_ocr". The knowledge base stores
|
||||
// them per pipeline rather than in one flat map, and the panel renders one
|
||||
// control per field without naming any pipeline.
|
||||
export interface ImagePipelineField {
|
||||
key: string;
|
||||
type: 'bool' | 'string' | 'enum';
|
||||
label: string;
|
||||
description: string;
|
||||
default: unknown;
|
||||
options: string[] | null;
|
||||
/** Whether this field turns an action on or off, as opposed to tuning one. */
|
||||
decides_action?: boolean;
|
||||
}
|
||||
|
||||
/** One condition the chosen pipeline places on its own tunables. The backend
|
||||
* owns the rules — including the wording each one triggers — so the panel never
|
||||
* decides on its own what a set of settings may or may not be. */
|
||||
export interface ImagePipelineRule {
|
||||
at_least_one: string[];
|
||||
/** i18n key of this rule's own message, translated at render time. */
|
||||
message_key: string;
|
||||
/** Which control to point at; defaults to the first of at_least_one. */
|
||||
field?: string;
|
||||
}
|
||||
|
||||
export interface ImagePipelineSpec {
|
||||
id: string;
|
||||
name: string;
|
||||
/** How the pipeline works, shown under the pick; i18n overlays on top. */
|
||||
description: string;
|
||||
fields: ImagePipelineField[];
|
||||
/** Conditions over those fields. Empty means the pipeline accepts any of it. */
|
||||
rules?: ImagePipelineRule[];
|
||||
}
|
||||
|
||||
export interface ImagePipelineViolation {
|
||||
message_key: string;
|
||||
field?: string;
|
||||
}
|
||||
|
||||
/** POST /api/v1/image-pipelines/validate: every rule of the named pipeline that
|
||||
* the given tunables break, translated by the caller rather than sent as prose.
|
||||
* `valid: true` means the backend found none. */
|
||||
export async function validateImagePipelines(
|
||||
pipelineId: string,
|
||||
params: Record<string, unknown>,
|
||||
): Promise<{ valid: boolean; violations: ImagePipelineViolation[] }> {
|
||||
const res = await post<{
|
||||
success: boolean;
|
||||
data: { valid: boolean; violations: ImagePipelineViolation[] };
|
||||
}>(`/api/v1/image-pipelines/validate`, { pipeline_id: pipelineId, params });
|
||||
return res.data;
|
||||
}
|
||||
|
||||
// Fetch every registered image pipeline. The backend owns the list, so a
|
||||
// pipeline added there shows up here as an option with no frontend change. The
|
||||
// registry is global, not per-KB, hence the top-level route.
|
||||
export async function fetchImagePipelines(): Promise<ImagePipelineSpec[]> {
|
||||
const res = await get<{ success: boolean; data: ImagePipelineSpec[] }>(
|
||||
`/api/v1/image-pipelines`,
|
||||
);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
export type VectorStoreSource = 'env' | 'user' | 'shared' | 'unavailable';
|
||||
export type VectorStoreStatus = 'available' | 'unavailable';
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,12 @@ const EXTRA_PREFIXES = [
|
|||
// so it is registered as a whole prefix: a new backend attribute must keep
|
||||
// being translated without touching the audit.
|
||||
'imageAttr.',
|
||||
// Image-pipeline panel text is keyed by the registry response too
|
||||
// (imagePipeline.smartocr.label …), likewise runtime-built: a
|
||||
// pipeline or field added in the backend must keep being translated without
|
||||
// touching the audit. The panel's own wording is literal and lives in the
|
||||
// locale files under knowledgeEditor.advanced.multimodal.imagePipeline*.
|
||||
'imagePipeline.',
|
||||
// Gallery attribute labels are keyed by the namespaced attribute id from
|
||||
// the gallery contract, likewise runtime-built; static labels for the few
|
||||
// builtin attributes ship in the locale files as
|
||||
|
|
|
|||
|
|
@ -4015,6 +4015,14 @@ export default {
|
|||
imageVectorModelUnsupported: 'The current embedding model does not declare image input, so no image vectors will be indexed or recalled',
|
||||
imageAttrsSchemaLabel: 'Observable image attributes',
|
||||
imageAttrsSchemaDescription: 'The model observes the attributes below (defined by the backend registry) to drive the OCR policy',
|
||||
imagePipelineSectionLabel: 'Image parsing scheme',
|
||||
imagePipelineSectionDescription:
|
||||
'With multimodal parsing on, pick how images are parsed.',
|
||||
imagePipelineLabel: 'Scheme',
|
||||
imagePipelinePlaceholder: 'Select a scheme',
|
||||
imagePipelineLoading: 'Loading pipelines…',
|
||||
imagePipelineLoadError: 'Could not load pipelines',
|
||||
imagePipelineValidateError: 'The current settings could not be checked; the backend re-checks them on save.',
|
||||
imageAttrsOcrConditions: 'Trigger OCR based on the observed attribute conditions',
|
||||
imageAttrsOcrConditionsDesc: 'When the observed attributes match the conditions below, OCR runs on the image',
|
||||
imageAttrsOcrOnUnobserved: 'Run OCR when image-attribute observation fails',
|
||||
|
|
@ -8113,5 +8121,55 @@ export default {
|
|||
'false': { label: 'No', description: 'a photo, drawing, icon or decoration' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// Overlay wording for the pipeline panel, indexed by pipeline id and field
|
||||
// key (the backend registry supplies both; this only translates). Anything
|
||||
// not covered here falls back to the backend's own wording, so a new
|
||||
// pipeline never shows up as a bare key.
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: 'Turn on at least one parsing action, otherwise nothing will be processed.',
|
||||
},
|
||||
default: {
|
||||
name: 'Manual',
|
||||
description: 'Turn the parsing actions on or off to fit the task.',
|
||||
enable_caption: {
|
||||
label: 'Image caption',
|
||||
description: 'Ask the model for a one-line description of every image and store it as the caption.'
|
||||
},
|
||||
enable_ocr: {
|
||||
label: 'OCR',
|
||||
description: 'Extract the text that appears in the image.'
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: 'Caption with thinking',
|
||||
description: 'Let the model think before it writes the caption. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the caption, failing the task. Only enable it when you have confirmed it is necessary.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: 'OCR with thinking',
|
||||
description: 'Let the model think before it transcribes the text. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the transcribed text, failing the task. Only enable it when you have confirmed it is necessary.',
|
||||
},
|
||||
}
|
||||
},
|
||||
smartocr: {
|
||||
name: 'Smart',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: 'Describe with thinking',
|
||||
description: 'Let the model think before it describes the image and reports its attributes. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the description, failing the task. Only enable it when you have confirmed it is necessary.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: 'Text recognition with thinking',
|
||||
description: 'Let the model think before it transcribes the text. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the transcribed text, failing the task. Only enable it when you have confirmed it is necessary.',
|
||||
},
|
||||
},
|
||||
description:
|
||||
'Observe the image attributes and describe the image first, then decide from the attributes whether to run OCR, saving model calls and speeding up parsing.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4015,6 +4015,14 @@ export default {
|
|||
imageVectorModelUnsupported: '現在の埋め込みモデルは画像入力を宣言していないため、画像ベクトルは生成も検索もされません',
|
||||
imageAttrsSchemaLabel: '観察可能な画像属性',
|
||||
imageAttrsSchemaDescription: 'モデルは以下の属性(バックエンドのレジストリで定義)を観察して OCR ポリシーを決めます',
|
||||
imagePipelineSectionLabel: '画像解析方式',
|
||||
imagePipelineSectionDescription:
|
||||
'マルチモーダル解析が有効な場合、画像の解析方式を選択できます。',
|
||||
imagePipelineLabel: '解析方式',
|
||||
imagePipelinePlaceholder: '解析方式を選択',
|
||||
imagePipelineLoading: 'パイプラインを読み込み中…',
|
||||
imagePipelineLoadError: 'パイプラインの読み込みに失敗しました',
|
||||
imagePipelineValidateError: '現在の設定を検証できませんでした。保存時にバックエンドが再度检查します。',
|
||||
imageAttrsOcrConditions: '観察した属性条件に基づいて OCR を実行',
|
||||
imageAttrsOcrConditionsDesc: '観察した属性が以下の条件を満たす場合、その画像に OCR を実行します',
|
||||
imageAttrsOcrOnUnobserved: '画像属性の観察に失敗した場合も OCR を実行',
|
||||
|
|
@ -8107,5 +8115,55 @@ export default {
|
|||
'false': { label: 'いいえ', description: 'いいえ —— 写真・イラスト・アイコン・装飾' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// パイプラインパネルの上書き文言:パイプライン id とフィールド key で索引
|
||||
// (どちらもバックエンドのレジストリが提供し、ここでは翻訳のみ)。ここに
|
||||
// ない部分はバックエンド固有の文言にフォールバックするため、新しい
|
||||
// パイプラインがキー名のまま表示されることはありません。
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: '解析アクションを1つ以上オンにしてください。オフのままでは何も処理されません。',
|
||||
},
|
||||
default: {
|
||||
name: 'マニュアル',
|
||||
description: 'タスクに合わせて解析アクションのスイッチを手動でオン/オフします。',
|
||||
enable_caption: {
|
||||
label: '画像説明',
|
||||
description: '各画像について一言の説明を生成し、キャプションとして保存します。'
|
||||
},
|
||||
enable_ocr: {
|
||||
label: '文字認識',
|
||||
description: '画像内に出現するテキストを抽出します。'
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '説明時に思考を有効',
|
||||
description: '説明を書く前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字認識時に思考を有効',
|
||||
description: 'テキストを抽出する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。',
|
||||
},
|
||||
}
|
||||
},
|
||||
smartocr: {
|
||||
name: 'スマート',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '説明時に思考を有効',
|
||||
description: '画像を説明し属性を報告する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字認識時に思考を有効',
|
||||
description: 'テキストを抽出する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。',
|
||||
},
|
||||
},
|
||||
description:
|
||||
'まず画像の特徴を観察して説明を生成し、その特徴に基づいて OCR を実行するかどうかを判断します。モデル呼び出しを節約し、解析速度を向上させます。'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4362,6 +4362,14 @@ export default {
|
|||
imageVectorModelUnsupported: '현재 임베딩 모델은 이미지 입력을 선언하지 않아 이미지 벡터가 생성되거나 검색되지 않습니다',
|
||||
imageAttrsSchemaLabel: '관찰 가능한 이미지 속성',
|
||||
imageAttrsSchemaDescription: '모델은 아래 속성(백엔드 레지스트리 정의)을 관찰해 OCR 정책을 결정합니다',
|
||||
imagePipelineSectionLabel: '이미지 분석 방식',
|
||||
imagePipelineSectionDescription:
|
||||
'멀티모달 분석이 켜져 있으면 이미지의 분석 방식을 필요에 따라 선택할 수 있습니다.',
|
||||
imagePipelineLabel: '분석 방식',
|
||||
imagePipelinePlaceholder: '분석 방식 선택',
|
||||
imagePipelineLoading: '파이프라인을 불러오는 중…',
|
||||
imagePipelineLoadError: '파이프라인을 불러오지 못했습니다',
|
||||
imagePipelineValidateError: '현재 설정을 검증할 수 없습니다. 저장 시 백엔드가 다시 검사합니다.',
|
||||
imageAttrsOcrConditions: '관찰된 속성 조건에 따라 OCR 실행',
|
||||
imageAttrsOcrConditionsDesc: '관찰된 속성이 아래 조건을 충족하면 해당 이미지에 OCR을 실행합니다',
|
||||
imageAttrsOcrOnUnobserved: '이미지 속성 관찰에 실패해도 OCR 실행',
|
||||
|
|
@ -8107,5 +8115,55 @@ export default {
|
|||
'false': { label: '아니오', description: '아니오 —— 사진, 삽화, 아이콘, 장식' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// 파이프라인 패널의 오버레이 문구: 파이프라인 id와 필드 key로 색인합니다
|
||||
// (둘 다 백엔드 레지스트리가 제공하며 여기서는 번역만 합니다). 여기서
|
||||
// 다루지 않은 부분은 백엔드 자체 문구로 폴백되므로, 새 파이프라인이 키
|
||||
// 이름 그대로 표시되는 일은 없습니다.
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: '분석 동작을 하나 이상 켜세요. 모두 꺼져 있으면 아무것도 처리되지 않습니다.',
|
||||
},
|
||||
default: {
|
||||
name: '수동',
|
||||
description: '작업에 맞게 분석 동작 스위치를 직접 켜거나 끕니다.',
|
||||
enable_caption: {
|
||||
label: '이미지 설명',
|
||||
description: '각 이미지에 대한 한 줄 설명을 생성해 캡션으로 저장합니다.'
|
||||
},
|
||||
enable_ocr: {
|
||||
label: '문자 인식',
|
||||
description: '이미지 속에 나타나는 텍스트를 추출합니다.'
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '설명 시 사고 사용',
|
||||
description: '설명을 쓰기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: 'OCR 시 사고 사용',
|
||||
description: '텍스트를 추출하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.',
|
||||
},
|
||||
}
|
||||
},
|
||||
smartocr: {
|
||||
name: '스마트',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '설명 시 사고 사용',
|
||||
description: '이미지를 설명하고 속성을 보고하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '문자 인식 시 사고 사용',
|
||||
description: '텍스트를 추출하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.',
|
||||
},
|
||||
},
|
||||
description:
|
||||
'먼저 이미지 특징을 관찰하여 설명을 생성하고, 그 특징을 바탕으로 OCR 실행 여부를 결정합니다. 모델 호출을 절약하고 분석 속도를 높입니다.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4362,6 +4362,14 @@ export default {
|
|||
imageVectorModelUnsupported: 'Текущая модель эмбеддингов не заявляет поддержку изображений, поэтому векторы изображений не будут ни созданы, ни найдены',
|
||||
imageAttrsSchemaLabel: 'Наблюдаемые атрибуты изображений',
|
||||
imageAttrsSchemaDescription: 'Модель наблюдает перечисленные ниже атрибуты (определены реестром бэкенда), чтобы управлять политикой OCR',
|
||||
imagePipelineSectionLabel: 'Схема разбора изображений',
|
||||
imagePipelineSectionDescription:
|
||||
'Когда включён мультимодальный разбор, можно выбрать способ разбора изображений.',
|
||||
imagePipelineLabel: 'Способ разбора',
|
||||
imagePipelinePlaceholder: 'Выберите способ разбора',
|
||||
imagePipelineLoading: 'Загрузка пайплайнов…',
|
||||
imagePipelineLoadError: 'Не удалось загрузить пайплайны',
|
||||
imagePipelineValidateError: 'Текущие настройки не удалось проверить; бэкенд проверит их при сохранении.',
|
||||
imageAttrsOcrConditions: 'Запуск OCR по наблюдаемым условиям атрибутов',
|
||||
imageAttrsOcrConditionsDesc: 'Когда наблюдаемые атрибуты соответствуют условиям ниже, для изображения выполняется OCR',
|
||||
imageAttrsOcrOnUnobserved: 'Запускать OCR при сбое наблюдения атрибутов изображения',
|
||||
|
|
@ -8107,5 +8115,55 @@ export default {
|
|||
'false': { label: 'Нет', description: 'нет —— фото, рисунок, значок или декор' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// Текст панели пайплайнов: индексируется по id пайплайна и ключу поля
|
||||
// (оба даёт реестр бэкенда, здесь только перевод). Всё, что здесь не
|
||||
// указано, откатывается к формулировке бэкенда, поэтому новый пайплайн
|
||||
// не покажется голым именем ключа.
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: 'Включите хотя бы одно действие разбора, иначе ничего не будет обработано.',
|
||||
},
|
||||
default: {
|
||||
name: 'Ручной',
|
||||
description: 'Пользователь сам включает или выключает действия разбора под задачу.',
|
||||
enable_caption: {
|
||||
label: 'Описание изображения',
|
||||
description: 'Генерирует однострочное описание каждого изображения и сохраняет его как подпись.'
|
||||
},
|
||||
enable_ocr: {
|
||||
label: 'Распознавание текста',
|
||||
description: 'Извлекает текст, присутствующий на изображении.'
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: 'С размышлением при описании',
|
||||
description: 'Дать модели поразмышлять перед тем, как писать описание. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: 'С размышлением при распознавании',
|
||||
description: 'Дать модели поразмышлять перед расшифровкой текста. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.',
|
||||
},
|
||||
}
|
||||
},
|
||||
smartocr: {
|
||||
name: 'Умный',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: 'С размышлением при описании',
|
||||
description: 'Дать модели поразмышлять перед описанием изображения и отчётом о его признаках. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: 'С размышлением при распознавании текста',
|
||||
description: 'Дать модели поразмышлять перед расшифровкой текста. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.',
|
||||
},
|
||||
},
|
||||
description:
|
||||
'Сначала наблюдаются признаки изображения и создаётся его описание, затем по этим признакам решается, вызывать ли OCR. Экономит вызовы модели и ускоряет разбор.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4364,6 +4364,13 @@ export default {
|
|||
imageVectorModelUnsupported: '当前向量模型未声明图片输入,开启后不会生成或召回图片向量',
|
||||
imageAttrsSchemaLabel: '可观察的图片属性',
|
||||
imageAttrsSchemaDescription: '模型会观察以下属性(由后端注册表定义)以驱动 OCR 策略',
|
||||
imagePipelineSectionLabel: '图片解析方案',
|
||||
imagePipelineSectionDescription: '多模态解析开启后,可以根据需要选择图片的解析方案',
|
||||
imagePipelineLabel: '选择解析方案',
|
||||
imagePipelinePlaceholder: '请选择解析方案',
|
||||
imagePipelineLoading: '正在读取流水线…',
|
||||
imagePipelineLoadError: '读取流水线失败',
|
||||
imagePipelineValidateError: '无法校验当前设置,保存时后端会再次检查。',
|
||||
imageAttrsOcrConditions: '根据观察到的属性条件触发 OCR',
|
||||
imageAttrsOcrConditionsDesc: '当观察到的属性满足以下条件时,对图片进行 OCR',
|
||||
imageAttrsOcrOnUnobserved: '图片属性观察失败时仍执行 OCR',
|
||||
|
|
@ -8109,5 +8116,53 @@ export default {
|
|||
'false': { label: '否', description: '否 —— 照片、插画、图标或装饰图' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// 流水线选择面板的覆盖文案:按流水线 id 与字段 key 索引(后端注册表给出
|
||||
// id 与 key,这里只做翻译)。未覆盖的流水线或字段回落到后端自带的英文
|
||||
// 说明,所以新增流水线不会显示成键名。
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: '至少开启一个解析动作,否则多模态不会处理任何内容。',
|
||||
},
|
||||
default: {
|
||||
name: '传统',
|
||||
description: '用户根据任务需要手动选择解析动作开关。',
|
||||
enable_caption: {
|
||||
label: '图片描述',
|
||||
description: '让模型为每张图片生成一句内容描述,作为图片的说明文字。'
|
||||
},
|
||||
enable_ocr: {
|
||||
label: '文字识别',
|
||||
description: '提取图片中出现的文字。'
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '图片描述时开启思考',
|
||||
description: '让模型在写描述前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字识别时开启思考',
|
||||
description: '让模型在提取文字前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。',
|
||||
},
|
||||
}
|
||||
},
|
||||
smartocr: {
|
||||
name: '智能模式',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '描述时开启思考',
|
||||
description: '让模型在描述图片并观察属性前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字识别时开启思考',
|
||||
description: '让模型在提取文字前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。',
|
||||
},
|
||||
},
|
||||
description: '先观察图片特征并进行图片描述,再根据图片特征决定是否调用OCR对图片进行识别,以节省模型调用次数并提升解析速度。'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4368,6 +4368,13 @@ export default {
|
|||
imageAttrsOcrConditionsDesc: '當觀察到的屬性滿足以下條件時,對圖片進行 OCR',
|
||||
imageAttrsOcrOnUnobserved: '圖片屬性觀察失敗時仍執行 OCR',
|
||||
imageAttrsOcrOnUnobservedDesc: '當模型未能正確觀察到圖片屬性時,預設仍執行 OCR 兜底,以免漏掉正文文字;關閉則跳過。(採用 4B 等小參數視覺模型,或自定義的圖片解析提示語與系統提示語衝突時,可能造成觀察失敗;8B 及以上模型的失敗概率很低,不建議關閉)',
|
||||
imagePipelineSectionLabel: '圖片解析方案',
|
||||
imagePipelineSectionDescription: '多模態解析開啟後,可以根據需要選擇圖片的解析方案',
|
||||
imagePipelineLabel: '選擇解析方案',
|
||||
imagePipelinePlaceholder: '請選擇解析方案',
|
||||
imagePipelineLoading: '正在讀取流水線…',
|
||||
imagePipelineLoadError: '讀取流水線失敗',
|
||||
imagePipelineValidateError: '無法校驗當前設定,儲存時後端會再次檢查。',
|
||||
imagePipelineKbNote: '預設跟隨知識庫設定,可針對本次任務調整'
|
||||
},
|
||||
tableMetadataInstructions: {
|
||||
|
|
@ -8109,5 +8116,50 @@ export default {
|
|||
'false': { label: '否', description: '否 ── 照片、插畫、圖示或裝飾圖' }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
imagePipeline: {
|
||||
errors: {
|
||||
noActionEnabled: '至少開啟一個解析動作,否則多模態不會處理任何內容。',
|
||||
},
|
||||
default: {
|
||||
name: '傳統',
|
||||
description: '使用者根據任務需要手動選擇解析動作開關。',
|
||||
enable_caption: {
|
||||
label: '圖片描述',
|
||||
description: '讓模型為每張圖片生成一句內容描述,作為圖片的說明文字。',
|
||||
},
|
||||
enable_ocr: {
|
||||
label: '文字辨識',
|
||||
description: '提取圖片中出現的文字。',
|
||||
},
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '圖片描述時開啟思考',
|
||||
description: '讓模型在寫描述前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字辨識時開啟思考',
|
||||
description: '讓模型在提取文字前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。',
|
||||
},
|
||||
},
|
||||
},
|
||||
smartocr: {
|
||||
name: '智能模式',
|
||||
caption: {
|
||||
thinking: {
|
||||
label: '描述時開啟思考',
|
||||
description: '讓模型在描述圖片並觀察屬性前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。',
|
||||
},
|
||||
},
|
||||
ocr: {
|
||||
thinking: {
|
||||
label: '文字辨識時開啟思考',
|
||||
description: '讓模型在提取文字前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。',
|
||||
},
|
||||
},
|
||||
description: '先觀察圖片特徵並進行圖片描述,再根據圖片特徵決定是否呼叫 OCR 對圖片進行辨識,以節省模型呼叫次數並提升解析速度。',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,11 @@ export interface KnowledgeProcessOverrides {
|
|||
// Per-upload override of the knowledge base's attribute-observed image
|
||||
// pipeline switch.
|
||||
image_attrs_enabled?: boolean
|
||||
// Per-upload override of the image pipeline and its private tunables. The
|
||||
// pair travels as one unit; when image_pipeline is set the backend also
|
||||
// derives image_attrs_enabled from it.
|
||||
image_pipeline?: string
|
||||
image_pipeline_params?: Record<string, unknown>
|
||||
// Controls indexing for this document, without changing KB retrieval.
|
||||
image_vector_enabled?: boolean
|
||||
// Per-task attribute -> work table; merged per action key on top of the
|
||||
|
|
|
|||
|
|
@ -1,48 +1,93 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
|
||||
import { buildImageProcessingConfig } from './imageProcessingConfig.ts'
|
||||
import {
|
||||
buildImageProcessingConfig,
|
||||
buildPipelineFields,
|
||||
resolveImagePipelineFromKb,
|
||||
normalizeImagePipelineId,
|
||||
IMAGE_PIPELINE_SMARTOCR,
|
||||
LEGACY_PIPELINE_CAPTION_OCR,
|
||||
LEGACY_PIPELINE_OB_CAP_OCR,
|
||||
type ImageProcessingEdits,
|
||||
} from './imageProcessingConfig.ts'
|
||||
|
||||
const DEFAULT_ON = [
|
||||
{ prop: 'contain.text', is: 'block' },
|
||||
{ prop: 'contain.data_visual', is: 'true' },
|
||||
]
|
||||
|
||||
/** The edits every test shares; `edits` overrides them. */
|
||||
function editsWith(overrides: Partial<ImageProcessingEdits> = {}): ImageProcessingEdits {
|
||||
return {
|
||||
imageAttrsEnabled: true,
|
||||
imageVectorEnabled: false,
|
||||
onUnobserved: false,
|
||||
defaultOn: DEFAULT_ON,
|
||||
pipelineId: IMAGE_PIPELINE_SMARTOCR,
|
||||
pipelineParams: {},
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
test('saving keeps OCR conditions customised through the API', () => {
|
||||
// Regression: the editor always wrote the registry default into `on`, so an
|
||||
// unrelated edit replaced a KB's custom OCR conditions with the default table.
|
||||
const custom = [{ prop: 'contain.text', is: 'sparse' }]
|
||||
const built = buildImageProcessingConfig(
|
||||
{ model_id: 'vlm-1', image_attrs_enabled: true, image_actions: { ocr: { on: custom, on_unobserved: true } } },
|
||||
{ imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: false, defaultOn: DEFAULT_ON },
|
||||
editsWith(),
|
||||
)
|
||||
|
||||
assert.deepEqual(built, {
|
||||
model_id: 'vlm-1',
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: custom, on_unobserved: false } },
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
})
|
||||
})
|
||||
|
||||
test('a KB without custom conditions gets the registry default alongside on_unobserved', () => {
|
||||
const built = buildImageProcessingConfig(
|
||||
{ model_id: 'vlm-1' },
|
||||
{ imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: false, defaultOn: DEFAULT_ON },
|
||||
editsWith(),
|
||||
)
|
||||
|
||||
assert.deepEqual(built, {
|
||||
model_id: 'vlm-1',
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } },
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
})
|
||||
})
|
||||
|
||||
test('the chosen pipeline and its private parameters are written back', () => {
|
||||
const built = buildImageProcessingConfig(
|
||||
{ model_id: 'vlm-1' },
|
||||
editsWith({ pipelineId: LEGACY_PIPELINE_CAPTION_OCR, pipelineParams: { enable_caption: false, enable_ocr: true } }),
|
||||
)
|
||||
|
||||
// Both keys of this pipeline travel, and they are capitalised exactly as the
|
||||
// backend declares them: a misspelled key is silently ignored at run time.
|
||||
assert.deepEqual(built, {
|
||||
model_id: 'vlm-1',
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } },
|
||||
image_pipeline: LEGACY_PIPELINE_CAPTION_OCR,
|
||||
image_pipeline_params: { enable_caption: false, enable_ocr: true },
|
||||
})
|
||||
})
|
||||
|
||||
test('an unchanged configuration is not sent', () => {
|
||||
const custom = [{ prop: 'contain.text', is: 'sparse' }]
|
||||
const snapshot = { image_attrs_enabled: true, image_actions: { ocr: { on: custom, on_unobserved: true } } }
|
||||
const snapshot = {
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: custom, on_unobserved: true } },
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
}
|
||||
|
||||
assert.equal(
|
||||
buildImageProcessingConfig(snapshot, { imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON }),
|
||||
buildImageProcessingConfig(snapshot, editsWith({ onUnobserved: true })),
|
||||
null,
|
||||
)
|
||||
})
|
||||
|
|
@ -50,13 +95,13 @@ test('an unchanged configuration is not sent', () => {
|
|||
test('the image-vector switch is saved when turned on and when turned back off', () => {
|
||||
const on = buildImageProcessingConfig(
|
||||
{ model_id: 'vlm-1' },
|
||||
{ imageAttrsEnabled: false, imageVectorEnabled: true, onUnobserved: true, defaultOn: DEFAULT_ON },
|
||||
editsWith({ imageAttrsEnabled: false, imageVectorEnabled: true }),
|
||||
)
|
||||
assert.equal(on?.image_vector_enabled, true)
|
||||
|
||||
const off = buildImageProcessingConfig(
|
||||
{ model_id: 'vlm-1', image_vector_enabled: true },
|
||||
{ imageAttrsEnabled: false, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON },
|
||||
editsWith({ imageAttrsEnabled: false, imageVectorEnabled: false }),
|
||||
)
|
||||
assert.equal(off?.image_vector_enabled, false)
|
||||
})
|
||||
|
|
@ -64,11 +109,67 @@ test('the image-vector switch is saved when turned on and when turned back off',
|
|||
test('a knowledge base that never touched the image-vector switch does not get one written', () => {
|
||||
// The backend reads a missing key as off, which is what every knowledge
|
||||
// base from before the switch has; an unrelated save must leave it so.
|
||||
const snapshot = { image_attrs_enabled: true, image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: true } } }
|
||||
const snapshot = {
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: true } },
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
}
|
||||
assert.equal(
|
||||
buildImageProcessingConfig(snapshot, {
|
||||
imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON,
|
||||
}),
|
||||
buildImageProcessingConfig(snapshot, editsWith({ onUnobserved: true })),
|
||||
null,
|
||||
)
|
||||
})
|
||||
|
||||
test('clearing the pick removes the stored one rather than repeating it', () => {
|
||||
// An empty pick means "resolve from image_attrs_enabled", which is what the
|
||||
// field already said by being absent, so saving has to actually drop it —
|
||||
// along with the parameters, which belong to the pick that is gone.
|
||||
const snapshot = {
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
image_pipeline_params: { capture_caption: true },
|
||||
}
|
||||
const built = buildImageProcessingConfig(snapshot, editsWith({
|
||||
pipelineId: '',
|
||||
pipelineParams: { enable_caption: false, enable_ocr: false },
|
||||
}))
|
||||
|
||||
assert.deepEqual(built, {
|
||||
image_attrs_enabled: true,
|
||||
image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } },
|
||||
})
|
||||
})
|
||||
|
||||
test('both dialogs drop an empty pick and empty parameters the same way', () => {
|
||||
// Regression: the upload dialog used to post `image_pipeline_params: {}`,
|
||||
// which reads as "the caller turned everything off" once a pipeline starts
|
||||
// reading its own keys with a default pinned at the read site.
|
||||
// A real pick with nothing to tune keeps the pick and drops the params;
|
||||
// an absent pick falls back to the backend's own resolution.
|
||||
assert.deepEqual(buildPipelineFields(IMAGE_PIPELINE_SMARTOCR, {}), {
|
||||
image_pipeline: IMAGE_PIPELINE_SMARTOCR,
|
||||
})
|
||||
assert.deepEqual(buildPipelineFields('', { enable_ocr: true }), {})
|
||||
})
|
||||
|
||||
test('stored renames resolve to the ids the panel offers', () => {
|
||||
// Regression: `caption_ocr` predates the picker and `ob_cap_ocr` predates
|
||||
// the rename to match the label; a reparse of a document saved under either
|
||||
// spelling must land on a selectable option, not on an empty one.
|
||||
assert.equal(normalizeImagePipelineId(LEGACY_PIPELINE_CAPTION_OCR), 'default')
|
||||
assert.equal(normalizeImagePipelineId(LEGACY_PIPELINE_OB_CAP_OCR), IMAGE_PIPELINE_SMARTOCR)
|
||||
assert.equal(
|
||||
resolveImagePipelineFromKb({ image_processing_config: { image_pipeline: LEGACY_PIPELINE_OB_CAP_OCR } }),
|
||||
IMAGE_PIPELINE_SMARTOCR,
|
||||
)
|
||||
// Only the observation switch, no pick at all: the legacy rule still applies.
|
||||
assert.equal(
|
||||
resolveImagePipelineFromKb({ image_processing_config: { image_attrs_enabled: true } }),
|
||||
IMAGE_PIPELINE_SMARTOCR,
|
||||
)
|
||||
assert.equal(
|
||||
resolveImagePipelineFromKb({ image_processing_config: { image_attrs_enabled: false } }),
|
||||
'default',
|
||||
)
|
||||
// Nothing stored at all — the oldest knowledge bases.
|
||||
assert.equal(resolveImagePipelineFromKb({}), 'default')
|
||||
})
|
||||
|
|
|
|||
|
|
@ -4,6 +4,13 @@ export interface ImageAttrConditionLike {
|
|||
is: string
|
||||
}
|
||||
|
||||
/** Pipeline ids. Both dialogs compare against these instead of literals. */
|
||||
export const IMAGE_PIPELINE_DEFAULT = 'default'
|
||||
export const IMAGE_PIPELINE_SMARTOCR = 'smartocr'
|
||||
/** Spellings a stored config may still carry; normalized, never honoured. */
|
||||
export const LEGACY_PIPELINE_CAPTION_OCR = 'caption_ocr'
|
||||
export const LEGACY_PIPELINE_OB_CAP_OCR = 'ob_cap_ocr'
|
||||
|
||||
/** The fields of the KB editor that feed image_processing_config. */
|
||||
export interface ImageProcessingEdits {
|
||||
imageAttrsEnabled: boolean
|
||||
|
|
@ -12,6 +19,53 @@ export interface ImageProcessingEdits {
|
|||
onUnobserved: boolean
|
||||
/** The registry's current default OCR conditions (GET /image-attrs/schema). */
|
||||
defaultOn: ImageAttrConditionLike[]
|
||||
/** The pipeline picked in the panel; empty means multimodal is off and the fields go away. */
|
||||
pipelineId: string
|
||||
/** That pipeline's private tunables, keyed by field. */
|
||||
pipelineParams: Record<string, unknown>
|
||||
}
|
||||
|
||||
/** The pipeline fields an overrides payload carries, with the empties dropped. */
|
||||
export interface PipelineFields {
|
||||
image_pipeline?: string
|
||||
image_pipeline_params?: Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a stored pipeline id onto the id the panel offers. Both renames are
|
||||
* folded in here so the KB editor, the upload dialog and the reparse dialog
|
||||
* agree on what a saved value means; unknown ids pass through and the caller
|
||||
* falls back. Mirrors backend types.NormalizeImagePipelineID.
|
||||
*/
|
||||
export function normalizeImagePipelineId(id: string | null | undefined): string {
|
||||
if (!id) return ''
|
||||
if (id === LEGACY_PIPELINE_CAPTION_OCR) return IMAGE_PIPELINE_DEFAULT
|
||||
if (id === LEGACY_PIPELINE_OB_CAP_OCR) return IMAGE_PIPELINE_SMARTOCR
|
||||
return id
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the pipeline half of a payload. An empty pick is not a value of its
|
||||
* own: the backend resolves from image_attrs_enabled when the field is absent,
|
||||
* so the pair is dropped rather than written as an empty pair. The same goes
|
||||
* for parameters — a pick with nothing to tune must not send an empty object,
|
||||
* which would read as "the caller turned everything off".
|
||||
*
|
||||
* This is the only place that decides emptiness, so the knowledge base editor
|
||||
* (whole-config replacement) and the upload/reparse dialogs (per-task
|
||||
* overrides) cannot drift apart on what "no setting" means.
|
||||
*/
|
||||
export function buildPipelineFields(
|
||||
pipelineId: string,
|
||||
pipelineParams: Record<string, unknown> | null | undefined,
|
||||
): PipelineFields {
|
||||
if (!pipelineId) return {}
|
||||
const fields: PipelineFields = {}
|
||||
if (pipelineId) fields.image_pipeline = pipelineId
|
||||
if (pipelineParams && Object.keys(pipelineParams).length > 0) {
|
||||
fields.image_pipeline_params = pipelineParams
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -25,6 +79,9 @@ export interface ImageProcessingEdits {
|
|||
* through the API keeps it; one without a custom list gets the registry's
|
||||
* current default, so it still follows the default as it evolves.
|
||||
*
|
||||
* The pipeline and its parameters are written too. They are the panel's own
|
||||
* settings; the observation switch is kept in step with the pick so a base that
|
||||
* was edited through the UI still reads the same way through the older switch.
|
||||
* image_vector_enabled is written only when it is on or was already stored:
|
||||
* a knowledge base that never touched the switch reads as off on the
|
||||
* backend, so writing `false` into it would be a change for nothing.
|
||||
|
|
@ -46,8 +103,32 @@ export function buildImageProcessingConfig(
|
|||
},
|
||||
},
|
||||
}
|
||||
// The pick carries the panel's own answer, so it replaces whatever the
|
||||
// snapshot had — including by removing it.
|
||||
const pipeline = buildPipelineFields(edits.pipelineId, edits.pipelineParams)
|
||||
if (pipeline.image_pipeline) built.image_pipeline = pipeline.image_pipeline
|
||||
else delete built.image_pipeline
|
||||
if (pipeline.image_pipeline_params) built.image_pipeline_params = pipeline.image_pipeline_params
|
||||
else delete built.image_pipeline_params
|
||||
if (edits.imageVectorEnabled || 'image_vector_enabled' in snap) {
|
||||
built.image_vector_enabled = edits.imageVectorEnabled
|
||||
}
|
||||
return JSON.stringify(built) === JSON.stringify(snap) ? null : built
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the pipeline id a knowledge base's stored config points at, by the
|
||||
* same rule the backend's ResolveImagePipelineID follows: an explicitly stored
|
||||
* id wins (a pre-rename "caption_ocr" normalizes to "default", "ob_cap_ocr" to
|
||||
* "smartocr"), otherwise the legacy observation switch picks smartocr when on
|
||||
* and default otherwise. Bases saved before the selector existed have no
|
||||
* config at all and land on default, so the picker always shows a real option.
|
||||
*/
|
||||
export function resolveImagePipelineFromKb(
|
||||
kb: Record<string, any> | null | undefined,
|
||||
): string {
|
||||
const cfg = kb?.image_processing_config
|
||||
const stored = normalizeImagePipelineId((cfg?.image_pipeline as string | undefined) || '')
|
||||
if (stored) return stored
|
||||
return cfg?.image_attrs_enabled ? IMAGE_PIPELINE_SMARTOCR : IMAGE_PIPELINE_DEFAULT
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,185 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { runInNewContext } from 'node:vm'
|
||||
import test from 'node:test'
|
||||
import { compileScript, parse } from '@vue/compiler-sfc'
|
||||
import ts from 'typescript'
|
||||
import { createRenderer, h, nextTick, reactive } from 'vue'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const filename = fileURLToPath(new URL('./ImagePipelineSettings.vue', import.meta.url))
|
||||
const { descriptor } = parse(readFileSync(filename, 'utf8'), { filename })
|
||||
const script = compileScript(descriptor, { id: 'image-pipeline-settings-test' }).content
|
||||
.replace('__expose();', '')
|
||||
.replace('return __returned__', '__expose(__returned__); return __returned__')
|
||||
const compiled = ts.transpileModule(script, {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText
|
||||
|
||||
const violation = { message_key: 'imagePipeline.errors.noActionEnabled', field: 'enable_caption' }
|
||||
const flushUpdates = async () => {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
await nextTick()
|
||||
}
|
||||
|
||||
type ValidationRequest = {
|
||||
pipelineId: string
|
||||
params: Record<string, unknown>
|
||||
resolve: (value: { violations: typeof violation[] }) => void
|
||||
reject: (reason: Error) => void
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const requests: ValidationRequest[] = []
|
||||
const timers = new Map<number, () => void>()
|
||||
let timerID = 0
|
||||
const exports: any = {}
|
||||
runInNewContext(compiled, {
|
||||
exports, console: { ...console, error() {} },
|
||||
setTimeout(callback: () => void) { timers.set(++timerID, callback); return timerID },
|
||||
clearTimeout(id: number) { timers.delete(id) },
|
||||
require(name: string) {
|
||||
if (name === 'vue') return require('vue')
|
||||
if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key, te: () => false }) }
|
||||
if (name === '@/api/knowledge-base') return {
|
||||
fetchImagePipelines: async () => [{ id: 'default', fields: [] }, { id: 'smartocr', fields: [] }],
|
||||
validateImagePipelines: (pipelineId: string, params: Record<string, unknown>) =>
|
||||
new Promise((resolve, reject) => requests.push({ pipelineId, params, resolve, reject })),
|
||||
}
|
||||
throw new Error(`Unexpected import: ${name}`)
|
||||
},
|
||||
})
|
||||
exports.default.render = () => null
|
||||
const renderer = createRenderer<any, any>({
|
||||
createElement: () => ({}), createText: () => ({}), createComment: () => ({}),
|
||||
insert() {}, remove() {}, setElementText() {}, setText() {}, patchProp() {},
|
||||
parentNode: () => null, nextSibling: () => null,
|
||||
})
|
||||
const props = reactive({ pipelineId: 'default', params: { enable_caption: false, enable_ocr: false } as Record<string, unknown> })
|
||||
const emitted: boolean[] = []
|
||||
let vm: any
|
||||
const app = renderer.createApp({ setup: () => () => h(exports.default, {
|
||||
...props,
|
||||
ref: (value: any) => { vm = value },
|
||||
'onUpdate:params': (params: Record<string, unknown>) => { props.params = params },
|
||||
'onUpdate:pipelineId': (pipelineId: string) => { props.pipelineId = pipelineId },
|
||||
'onUpdate:invalid': (invalid: boolean) => { emitted.push(invalid) },
|
||||
}) })
|
||||
app.mount({})
|
||||
await flushUpdates()
|
||||
return {
|
||||
requests, timers, emitted,
|
||||
get invalid() { return emitted.at(-1) },
|
||||
get validationFailed() { return vm.validationFailed },
|
||||
async setCaption(enabled: boolean) { vm.onParamChange('enable_caption', enabled); await flushUpdates() },
|
||||
async setPipeline(id: string) { vm.onPipelineChange(id); await flushUpdates() },
|
||||
async validate() {
|
||||
for (const [id, callback] of [...timers]) { timers.delete(id); callback() }
|
||||
await flushUpdates()
|
||||
},
|
||||
dispose: () => app.unmount(),
|
||||
}
|
||||
}
|
||||
|
||||
test('a late invalid response cannot block saving newer valid parameters', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
await f.validate()
|
||||
await f.setCaption(true)
|
||||
await f.validate()
|
||||
assert.equal(f.requests.length, 2)
|
||||
assert.equal(f.requests[0].params.enable_caption, false)
|
||||
assert.equal(f.requests[1].params.enable_caption, true)
|
||||
f.requests[1].resolve({ violations: [] })
|
||||
await flushUpdates()
|
||||
f.requests[0].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, false)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
||||
for (const response of ['valid', 'error'] as const) {
|
||||
test(`a stale ${response} response cannot erase the current invalid verdict`, async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
await f.setCaption(true)
|
||||
await f.validate()
|
||||
await f.setCaption(false)
|
||||
await f.validate()
|
||||
f.requests[1].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, true)
|
||||
if (response === 'valid') f.requests[0].resolve({ violations: [] })
|
||||
else f.requests[0].reject(new Error('old request failed'))
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, true)
|
||||
assert.equal(f.validationFailed, false)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
}
|
||||
|
||||
test('parameter changes invalidate responses even before the next debounce fires', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
await f.validate()
|
||||
await f.setCaption(true)
|
||||
assert.equal(f.requests.length, 1)
|
||||
f.requests[0].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, false)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
||||
test('switching pipelines clears the previous verdict and ignores its pending request', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
await f.validate()
|
||||
f.requests[0].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, true)
|
||||
await f.setCaption(true)
|
||||
await f.validate()
|
||||
await f.setPipeline('smartocr')
|
||||
assert.equal(f.invalid, false)
|
||||
await f.validate()
|
||||
assert.equal(f.requests[2].pipelineId, 'smartocr')
|
||||
assert.deepEqual({ ...f.requests[2].params }, {})
|
||||
f.requests[2].resolve({ violations: [] })
|
||||
f.requests[1].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, false)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
||||
test('unmount cancels scheduled validation and ignores in-flight results', async () => {
|
||||
const f = await fixture()
|
||||
await f.validate()
|
||||
await f.setCaption(true)
|
||||
const emitted = [...f.emitted]
|
||||
f.dispose()
|
||||
assert.equal(f.timers.size, 0)
|
||||
f.requests[0].resolve({ violations: [violation] })
|
||||
await flushUpdates()
|
||||
assert.deepEqual(f.emitted, emitted)
|
||||
await f.validate()
|
||||
assert.equal(f.requests.length, 1)
|
||||
})
|
||||
|
||||
test('a current validation failure still permits saving and is cleared by the next result', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
await f.validate()
|
||||
f.requests[0].reject(new Error('network unavailable'))
|
||||
await flushUpdates()
|
||||
assert.equal(f.invalid, false)
|
||||
assert.equal(f.validationFailed, true)
|
||||
await f.setCaption(true)
|
||||
await f.validate()
|
||||
f.requests[1].resolve({ violations: [] })
|
||||
await flushUpdates()
|
||||
assert.equal(f.validationFailed, false)
|
||||
assert.equal(f.invalid, false)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
|
@ -0,0 +1,334 @@
|
|||
<script setup lang="ts">
|
||||
/**
|
||||
* The image-pipeline panel.
|
||||
*
|
||||
* The list of pipelines and of their fields comes from the backend
|
||||
* (GET /api/v1/image-pipelines), so this component names no pipeline and knows
|
||||
* no field: adding one there arrives here as a new option and a new control.
|
||||
* A field is rendered by its declared type, and its label falls back to the
|
||||
* backend's own wording when this locale has nothing for it.
|
||||
*/
|
||||
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import {
|
||||
fetchImagePipelines,
|
||||
validateImagePipelines,
|
||||
type ImagePipelineField,
|
||||
type ImagePipelineSpec,
|
||||
type ImagePipelineViolation,
|
||||
} from '@/api/knowledge-base'
|
||||
|
||||
const props = defineProps<{
|
||||
/** The selected pipeline id. */
|
||||
pipelineId: string
|
||||
/** That pipeline's private tunables; a missing key reads the field default. */
|
||||
params: Record<string, unknown>
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:pipelineId': [value: string]
|
||||
'update:params': [value: Record<string, unknown>]
|
||||
/** Turned on while the current pick is configured to do nothing at all. */
|
||||
'update:invalid': [value: boolean]
|
||||
}>()
|
||||
|
||||
const { t, te } = useI18n()
|
||||
|
||||
const pipelines = ref<ImagePipelineSpec[]>([])
|
||||
const loading = ref(false)
|
||||
const loadError = ref('')
|
||||
|
||||
/** The pipeline currently selected, if the registry still carries it. */
|
||||
const current = computed<ImagePipelineSpec | undefined>(() =>
|
||||
pipelines.value.find((pipeline) => pipeline.id === props.pipelineId),
|
||||
)
|
||||
|
||||
/** Its fields, in the order the backend declared them. */
|
||||
const fields = computed<ImagePipelineField[]>(() => current.value?.fields ?? [])
|
||||
|
||||
// Every rule the selected pipeline breaks, reported by the backend and in the
|
||||
// backend's own terms: each violation carries the i18n key of its own wording
|
||||
// and the name of the control to point at. Nothing here decides what is
|
||||
// forbidden — the rules travel with the spec, so adding one to a pipeline needs
|
||||
// no change to this file, and a pipeline whose actions are not the user's to
|
||||
// switch off declares none and can never land in this list.
|
||||
const violations = ref<ImagePipelineViolation[]>([])
|
||||
const validationFailed = ref(false)
|
||||
|
||||
const invalid = computed<boolean>(() => violations.value.length > 0)
|
||||
|
||||
watch(invalid, (value) => emit('update:invalid', value), { immediate: true })
|
||||
|
||||
/** The debounce keeps a dribbled change from posting on every keystroke. */
|
||||
let validationTimer: ReturnType<typeof setTimeout> | undefined
|
||||
let validationGeneration = 0
|
||||
let disposed = false
|
||||
|
||||
function scheduleValidation() {
|
||||
if (disposed) return
|
||||
// Invalidate in-flight responses as soon as the inputs change, including
|
||||
// the interval before the next debounced request starts.
|
||||
validationGeneration++
|
||||
violations.value = []
|
||||
validationFailed.value = false
|
||||
if (validationTimer) clearTimeout(validationTimer)
|
||||
validationTimer = setTimeout(() => void validate(), 300)
|
||||
}
|
||||
|
||||
async function validate() {
|
||||
if (disposed) return
|
||||
const generation = ++validationGeneration
|
||||
const spec = current.value
|
||||
if (!spec) {
|
||||
violations.value = []
|
||||
return
|
||||
}
|
||||
try {
|
||||
const result = await validateImagePipelines(spec.id, props.params ?? {})
|
||||
if (generation !== validationGeneration) return
|
||||
violations.value = result.violations ?? []
|
||||
validationFailed.value = false
|
||||
} catch (error) {
|
||||
if (generation !== validationGeneration) return
|
||||
// An unreachable check is not a verdict. The panel must not block saving on
|
||||
// a network hiccup, and the save path asks the backend again anyway.
|
||||
violations.value = []
|
||||
validationFailed.value = true
|
||||
console.error('[ImagePipelineSettings] validation request failed', error)
|
||||
}
|
||||
}
|
||||
|
||||
watch(() => [props.pipelineId, props.params], scheduleValidation)
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
disposed = true
|
||||
validationGeneration++
|
||||
if (validationTimer) clearTimeout(validationTimer)
|
||||
})
|
||||
|
||||
/**
|
||||
* A field's declared default. The backend sends JSON, so an unset key arrives
|
||||
* as undefined; the switch below must still read a value rather than a blank.
|
||||
*/
|
||||
function defaultValue(field: ImagePipelineField): unknown {
|
||||
return field.default ?? false
|
||||
}
|
||||
|
||||
function paramValue(field: ImagePipelineField): unknown {
|
||||
const own = props.params?.[field.key]
|
||||
return own === undefined || own === null ? defaultValue(field) : own
|
||||
}
|
||||
|
||||
/** i18n overlay key for one field of one pipeline. */
|
||||
function fieldKey(field: ImagePipelineField, part: 'label' | 'description'): string {
|
||||
return `imagePipeline.${current.value?.id ?? ''}.${field.key}.${part}`
|
||||
}
|
||||
|
||||
function fieldLabel(field: ImagePipelineField): string {
|
||||
const key = fieldKey(field, 'label')
|
||||
return te(key) ? t(key) : field.label || field.key
|
||||
}
|
||||
|
||||
function fieldDescription(field: ImagePipelineField): string {
|
||||
const key = fieldKey(field, 'description')
|
||||
return te(key) ? t(key) : field.description ?? ''
|
||||
}
|
||||
|
||||
function pipelineName(pipeline: ImagePipelineSpec): string {
|
||||
const key = `imagePipeline.${pipeline.id}.name`
|
||||
return te(key) ? t(key) : pipeline.name
|
||||
}
|
||||
|
||||
/** How the selected pipeline works, shown under the pick. */
|
||||
function pipelineDescription(pipeline: ImagePipelineSpec): string {
|
||||
const key = `imagePipeline.${pipeline.id}.description`
|
||||
return te(key) ? t(key) : pipeline.description ?? ''
|
||||
}
|
||||
|
||||
function onPipelineChange(value: string) {
|
||||
emit('update:pipelineId', value)
|
||||
// A pipeline's parameters are only meaningful to that pipeline. Switching one
|
||||
// drops the other's, rather than carrying a stale key across that it silently
|
||||
// ignores.
|
||||
emit('update:params', {})
|
||||
}
|
||||
|
||||
function onParamChange(key: string, value: unknown) {
|
||||
emit('update:params', { ...props.params, [key]: value })
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
loadError.value = ''
|
||||
try {
|
||||
pipelines.value = await fetchImagePipelines()
|
||||
} catch (error) {
|
||||
loadError.value = error instanceof Error ? error.message : String(error)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
// The registry arrived, so the current pick can now be checked against it;
|
||||
// before that there was no rule to check it against.
|
||||
scheduleValidation()
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
// A dialog may mount before the KB it edits is loaded, so the selection can
|
||||
// arrive after the registry it must be validated against.
|
||||
watch(() => props.pipelineId, () => {
|
||||
if (!loading.value && pipelines.value.length === 0) load()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="image-pipeline-settings">
|
||||
<div v-if="loading" class="image-pipeline-hint">
|
||||
{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineLoading') }}
|
||||
</div>
|
||||
|
||||
<div v-else-if="loadError" class="image-pipeline-hint image-pipeline-hint--error">
|
||||
{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineLoadError') }}:{{ loadError }}
|
||||
</div>
|
||||
|
||||
<template v-else>
|
||||
<div class="image-pipeline-row">
|
||||
<div class="image-pipeline-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineLabel') }}</label>
|
||||
</div>
|
||||
<div class="image-pipeline-control">
|
||||
<t-select
|
||||
:value="pipelineId"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.imagePipelinePlaceholder')"
|
||||
@change="onPipelineChange"
|
||||
>
|
||||
<t-option
|
||||
v-for="pipeline in pipelines"
|
||||
:key="pipeline.id"
|
||||
:value="pipeline.id"
|
||||
:label="pipelineName(pipeline)"
|
||||
/>
|
||||
</t-select>
|
||||
</div>
|
||||
</div>
|
||||
<p v-if="current" class="image-pipeline-desc">{{ pipelineDescription(current) }}</p>
|
||||
|
||||
<!-- One row per field the running pipeline declares. A pipeline with no
|
||||
fields shows nothing here, which is the honest rendering of "nothing
|
||||
to tune" rather than an empty section. -->
|
||||
<div v-for="field in fields" :key="field.key" class="image-pipeline-row">
|
||||
<div class="image-pipeline-info">
|
||||
<label>{{ fieldLabel(field) }}</label>
|
||||
<p v-if="fieldDescription(field)" class="desc">{{ fieldDescription(field) }}</p>
|
||||
</div>
|
||||
<div class="image-pipeline-control">
|
||||
<t-switch
|
||||
v-if="field.type === 'bool'"
|
||||
:value="Boolean(paramValue(field))"
|
||||
size="medium"
|
||||
@change="(value: boolean) => onParamChange(field.key, value)"
|
||||
/>
|
||||
<t-select
|
||||
v-else-if="field.type === 'enum'"
|
||||
:value="String(paramValue(field) ?? '')"
|
||||
clearable
|
||||
@change="(value: string) => onParamChange(field.key, value)"
|
||||
>
|
||||
<t-option v-for="opt in field.options ?? []" :key="opt" :value="opt" :label="opt" />
|
||||
</t-select>
|
||||
<t-input
|
||||
v-else
|
||||
:value="String(paramValue(field) ?? '')"
|
||||
@change="(value: string) => onParamChange(field.key, value)"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Every broken rule, listed rather than summarised: each line names what
|
||||
is wrong and, through the field it points at, which control to turn
|
||||
back on. Sits at the foot of the panel, next to the button the user
|
||||
has to press to lose it. -->
|
||||
<ul v-if="invalid" class="image-pipeline-violations">
|
||||
<li
|
||||
v-for="violation in violations"
|
||||
:key="`${violation.message_key}:${violation.field ?? ''}`"
|
||||
class="image-pipeline-desc image-pipeline-desc--error"
|
||||
>
|
||||
{{ $t(violation.message_key) || violation.message_key }}
|
||||
</li>
|
||||
</ul>
|
||||
<p v-else-if="validationFailed" class="image-pipeline-desc">
|
||||
{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineValidateError') }}
|
||||
</p>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="less">
|
||||
.image-pipeline-settings {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
|
||||
// The section description above ends here; a wider gap marks where the
|
||||
// controls begin, matching the rhythm of the other setting rows.
|
||||
margin-top: 4px;
|
||||
|
||||
.image-pipeline-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.image-pipeline-info {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
|
||||
label {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.desc {
|
||||
margin: 4px 0 0;
|
||||
opacity: 0.75;
|
||||
font-size: var(--app-text-sm);
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.image-pipeline-control {
|
||||
flex: 0 0 200px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.image-pipeline-desc {
|
||||
margin: -4px 0 0;
|
||||
opacity: 0.75;
|
||||
font-size: var(--app-text-sm);
|
||||
line-height: 1.5;
|
||||
|
||||
&--error {
|
||||
margin: 0;
|
||||
opacity: 1;
|
||||
color: var(--error-color, #d54941);
|
||||
}
|
||||
}
|
||||
|
||||
.image-pipeline-hint {
|
||||
opacity: 0.75;
|
||||
font-size: var(--app-text-md);
|
||||
|
||||
&--error {
|
||||
color: var(--error-color, #d54941);
|
||||
}
|
||||
}
|
||||
|
||||
// One line per broken rule, kept compact so that several of them still read
|
||||
// as a footnote to the button rather than as a second form.
|
||||
.image-pipeline-violations {
|
||||
margin: 0;
|
||||
padding-left: 1em;
|
||||
list-style: disc;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
@ -402,12 +402,23 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="formData.multimodalConfig.enabled" class="setting-row">
|
||||
<div v-if="formData.multimodalConfig.enabled" class="setting-row setting-row-vertical">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.imageAttrsLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.imageAttrsDescription') }}</p>
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineSectionLabel') }}</label>
|
||||
<p class="desc">
|
||||
{{ $t('knowledgeEditor.advanced.multimodal.imagePipelineSectionDescription') }}
|
||||
</p>
|
||||
</div>
|
||||
<t-switch v-model="formData.imageAttrsEnabled" size="medium" />
|
||||
<!-- 面板取代了旧的观察开关:顶部多模态开关决定是否处理图片,
|
||||
这里决定怎么处理。选择器没有空选项,默认 default(内部名,
|
||||
UI 显示为「传统」);选中的方案同时写回 imageAttrsEnabled
|
||||
(只有智能模式点亮它),旧开关与属性面板跟随同一取值。 -->
|
||||
<ImagePipelineSettings
|
||||
v-model:params="formData.imagePipelineParams"
|
||||
:pipeline-id="formData.imagePipeline"
|
||||
@update:pipeline-id="onPipelineChange"
|
||||
@update:invalid="onPipelineInvalidChange"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div v-if="formData.multimodalConfig.enabled && formData.imageAttrsEnabled"
|
||||
|
|
@ -609,7 +620,12 @@ import {
|
|||
type ImageAttrSchema,
|
||||
type KnowledgeBaseProfile,
|
||||
} from '@/api/knowledge-base'
|
||||
import { buildImageProcessingConfig } from '@/utils/imageProcessingConfig'
|
||||
import {
|
||||
buildImageProcessingConfig,
|
||||
resolveImagePipelineFromKb as resolveKbImagePipeline,
|
||||
IMAGE_PIPELINE_DEFAULT,
|
||||
IMAGE_PIPELINE_SMARTOCR,
|
||||
} from '@/utils/imageProcessingConfig'
|
||||
import { imageAttrDisplay, imageAttrConditionDisplay } from '@/utils/imageAttrDisplay'
|
||||
|
||||
// The image-attribute registry, fetched from the backend (single source of
|
||||
|
|
@ -643,6 +659,7 @@ import GraphSettings from './settings/GraphSettings.vue'
|
|||
import KBShareSettings from './settings/KBShareSettings.vue'
|
||||
import DataSourceSettings from './settings/DataSourceSettings.vue'
|
||||
import KnowledgeBaseActivitySettings from './settings/KnowledgeBaseActivitySettings.vue'
|
||||
import ImagePipelineSettings from './ImagePipelineSettings.vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
const uiStore = useUIStore()
|
||||
|
|
@ -949,14 +966,19 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
|||
descriptionLanguage: '',
|
||||
customInstructions: ''
|
||||
},
|
||||
// 图片属性观察管线:开关 + on_unobserved 兜底切换由 UI 编辑;其余配置
|
||||
// (model_id 等)按加载时的快照原样回传,避免把 API 侧写入的设置洗掉。
|
||||
// 新建模式也必须用完整默认动作初始化——imageActions.ocr.on_unobserved
|
||||
// 直接被开关绑定,缺省会让打开开关的瞬间渲染崩溃。
|
||||
// 图片解析方案:方案选择与各方案的私有参数由 ImagePipelineSettings
|
||||
// 编辑;其余配置(model_id 等)按加载时的快照原样回传,避免把 API 侧写入
|
||||
// 的设置洗掉。新建模式也必须用完整默认动作初始化——imageActions.ocr
|
||||
// .on_unobserved 直接被开关绑定,缺省会让打开开关的瞬间渲染崩溃。
|
||||
imageAttrsEnabled: false,
|
||||
// 图片向量默认关闭:能处理图片的向量模型也常只用来编码文本。
|
||||
imageVectorEnabled: false,
|
||||
imageActions: mergeImageActions(),
|
||||
// 方案没有「留空」选项:多模态开关决定是否处理图片,开了就一定要有
|
||||
// 方案可跑,所以默认落在 default(内部名,UI 显示为「传统」)。选中的
|
||||
// 方案决定旧观察开关的取值(见 onPipelineChange)。
|
||||
imagePipeline: IMAGE_PIPELINE_DEFAULT,
|
||||
imagePipelineParams: {} as Record<string, unknown>,
|
||||
imageProcessingConfigSnapshot: null as Record<string, unknown> | null,
|
||||
asrConfig: {
|
||||
enabled: false,
|
||||
|
|
@ -1107,13 +1129,24 @@ const loadKBData = async (
|
|||
descriptionLanguage: kb.vlm_config?.description_language || '',
|
||||
customInstructions: kb.vlm_config?.custom_instructions || ''
|
||||
},
|
||||
imageAttrsEnabled:
|
||||
!!(kb as Record<string, any>).image_processing_config?.image_attrs_enabled,
|
||||
// 旧库迁移:本选择器出现之前保存的知识库没有 image_pipeline,按后端
|
||||
// ResolveImagePipelineID 的同一规则落到具体方案——观察开关开着就是
|
||||
// smartocr,其余(包括根本没有 image_processing_config 的老库)落到
|
||||
// default。存过的 id 还可能是更名前的 caption_ocr / ob_cap_ocr,
|
||||
// 归一化规则在 resolveKbImagePipeline 里,与上传/重新解析弹窗共用。
|
||||
imagePipeline: resolveKbImagePipeline(kb),
|
||||
// 旧观察开关与选中的方案保持同一个意思:只有智能模式把它点亮。
|
||||
imageAttrsEnabled: resolveKbImagePipeline(kb) === IMAGE_PIPELINE_SMARTOCR,
|
||||
imageVectorEnabled:
|
||||
!!(kb as Record<string, any>).image_processing_config?.image_vector_enabled,
|
||||
imageActions: mergeImageActions(
|
||||
(kb as Record<string, any>).image_processing_config?.image_actions,
|
||||
),
|
||||
imagePipelineParams:
|
||||
((kb as Record<string, any>).image_processing_config?.image_pipeline_params as Record<
|
||||
string,
|
||||
unknown
|
||||
>) ?? {},
|
||||
imageProcessingConfigSnapshot:
|
||||
(kb as Record<string, any>).image_processing_config || null,
|
||||
asrConfig: {
|
||||
|
|
@ -1295,6 +1328,29 @@ const handleMultimodalToggle = () => {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 知识库存量配置到方案 id 的解析规则已抽到 utils/imageProcessingConfig
|
||||
* (与后端 ResolveImagePipelineID 一致),供本弹窗与上传/重解析弹窗共用。
|
||||
*/
|
||||
|
||||
/**
|
||||
* 流水线参数的清理由 ImagePipelineSettings 自己完成(切换即清空上一条的
|
||||
* 私有参数);这里只负责让旧的观察开关跟上选择——只有观察属性流水线把它
|
||||
* 点亮,这样无论从哪个控件看,知识库读到的是同一个意思,下方的属性面板
|
||||
* 也跟随这个开关显隐。
|
||||
*/
|
||||
const onPipelineChange = (value: string) => {
|
||||
if (!formData.value) return
|
||||
formData.value.imagePipeline = value
|
||||
formData.value.imageAttrsEnabled = value === IMAGE_PIPELINE_SMARTOCR
|
||||
}
|
||||
|
||||
// 面板报上来的「选中的方案一个动作都没开」。只在多模态开启时阻止保存。
|
||||
const imagePipelineInvalid = ref(false)
|
||||
const onPipelineInvalidChange = (value: boolean) => {
|
||||
imagePipelineInvalid.value = value
|
||||
}
|
||||
|
||||
const handleMultimodalVLLMChange = (modelId: string) => {
|
||||
if (formData.value) {
|
||||
formData.value.multimodalConfig.vllmModelId = modelId
|
||||
|
|
@ -1441,6 +1497,15 @@ const validateForm = (): boolean => {
|
|||
return false
|
||||
}
|
||||
|
||||
// 选中的解析方案违反了它自己的规则(如传统方案两个动作开关都关):跑起来
|
||||
// 什么都不会做,属于配置错误而不是合法选择。提示文案与面板底部那几行同源,
|
||||
// 因此规则加得多、提示也就跟着变,这里不必跟着改。
|
||||
if (formData.value.multimodalConfig.enabled && imagePipelineInvalid.value) {
|
||||
MessagePlugin.warning(t('imagePipeline.errors.noActionEnabled'))
|
||||
currentSection.value = 'multimodal'
|
||||
return false
|
||||
}
|
||||
|
||||
if (formData.value.type === 'faq' && !formData.value.faqConfig?.indexMode) {
|
||||
MessagePlugin.warning(t('knowledgeEditor.messages.indexModeRequired'))
|
||||
currentSection.value = 'faq'
|
||||
|
|
@ -1508,6 +1573,10 @@ const buildSubmitData = () => {
|
|||
imageVectorEnabled: formData.value.imageVectorEnabled,
|
||||
onUnobserved: formData.value.imageActions.ocr.on_unobserved,
|
||||
defaultOn: displaySchema.value.default_actions.ocr.on,
|
||||
// 多模态开关决定是否处理图片:关闭时不写流水线字段(buildImageProcessingConfig
|
||||
// 对空 pipelineId 的处理是删掉这两个字段),配置里只留下观察开关的语义。
|
||||
pipelineId: formData.value.multimodalConfig.enabled ? formData.value.imagePipeline : '',
|
||||
pipelineParams: formData.value.imagePipelineParams,
|
||||
})
|
||||
if (built) {
|
||||
data.image_processing_config = built
|
||||
|
|
|
|||
|
|
@ -421,7 +421,7 @@
|
|||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="uiState.multimodalConfig.enabled" class="setting-row">
|
||||
<div v-if="uiState.multimodalConfig.enabled" class="setting-row setting-row-vertical">
|
||||
<div class="setting-info">
|
||||
<label>{{ t('uploadConfirm.imageEmbeddingModel') }}</label>
|
||||
<p class="desc">{{ t('uploadConfirm.imageIndexInherited') }}</p>
|
||||
|
|
@ -450,15 +450,22 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="uiState.multimodalConfig.enabled" class="setting-row">
|
||||
<div v-if="uiState.multimodalConfig.enabled" class="setting-row setting-row-vertical">
|
||||
<div class="setting-info">
|
||||
<label>{{ t('knowledgeEditor.advanced.multimodal.imageAttrsLabel') }}</label>
|
||||
<p class="desc">{{ t('knowledgeEditor.advanced.multimodal.imageAttrsDescription') }}</p>
|
||||
<label>{{ t('knowledgeEditor.advanced.multimodal.imagePipelineSectionLabel') }}</label>
|
||||
<p class="desc">{{ t('knowledgeEditor.advanced.multimodal.imagePipelineSectionDescription') }}</p>
|
||||
</div>
|
||||
<t-switch v-model="uiState.imageAttrsEnabled" size="medium" />
|
||||
<!-- 与知识库编辑器同款:方案与私有参数由注册表驱动,前端
|
||||
不写死任何方案名。选中智能模式时下方展开属性面板。 -->
|
||||
<ImagePipelineSettings
|
||||
v-model:params="uiState.imagePipelineParams"
|
||||
:pipeline-id="uiState.imagePipeline"
|
||||
@update:pipeline-id="uiState.imagePipeline = $event"
|
||||
@update:invalid="imagePipelineInvalid = $event"
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
v-if="uiState.multimodalConfig.enabled && uiState.imageAttrsEnabled"
|
||||
v-if="uiState.multimodalConfig.enabled && uiState.imagePipeline === IMAGE_PIPELINE_SMARTOCR"
|
||||
class="setting-row setting-row-vertical"
|
||||
>
|
||||
<div class="setting-info">
|
||||
|
|
@ -504,7 +511,7 @@
|
|||
.setting-info / .setting-control 都撑成 100% 宽,水平
|
||||
排布必然溢出,开关被顶到容器右缘之外。挪出来后与其它
|
||||
开关行共用同一套排版,右缘与「图片属性观察」对齐 -->
|
||||
<template v-if="uiState.multimodalConfig.enabled && uiState.imageAttrsEnabled">
|
||||
<template v-if="uiState.multimodalConfig.enabled && uiState.imagePipeline === IMAGE_PIPELINE_SMARTOCR">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ t('knowledgeEditor.advanced.multimodal.imageAttrsOcrOnUnobserved') }}</label>
|
||||
|
|
@ -681,6 +688,14 @@ import { getUploadFileKey } from '../utils/uploadSources'
|
|||
import { listKnowledgeTags, mergeImageActions, fetchImageAttrSchema, FALLBACK_IMAGE_ATTR_SCHEMA, type ImageActionsConfig, type ImageAttrSchema } from '@/api/knowledge-base'
|
||||
import { imageAttrDisplay, imageAttrConditionDisplay } from '@/utils/imageAttrDisplay'
|
||||
import KbUploadSourceDropdown from './KbUploadSourceDropdown.vue'
|
||||
import ImagePipelineSettings from '../ImagePipelineSettings.vue'
|
||||
import {
|
||||
buildPipelineFields,
|
||||
resolveImagePipelineFromKb,
|
||||
normalizeImagePipelineId,
|
||||
IMAGE_PIPELINE_DEFAULT,
|
||||
IMAGE_PIPELINE_SMARTOCR,
|
||||
} from '@/utils/imageProcessingConfig'
|
||||
import FolderPickerMenu, { type FolderOption } from './FolderPickerMenu.vue'
|
||||
import { folderOptionFromPath, sortFolderOptions } from '../folderTree'
|
||||
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
|
||||
|
|
@ -733,7 +748,11 @@ interface UploadUIState {
|
|||
summaryEnabled: boolean
|
||||
chunkingConfig: ChunkingUIConfig
|
||||
multimodalConfig: { enabled: boolean; vllmModelId: string; descriptionLanguage?: string; customInstructions?: string }
|
||||
imageAttrsEnabled: boolean
|
||||
// The image pipeline pick and its private tunables. The pick decides the
|
||||
// legacy image_attrs_enabled switch (only smartocr turns it on), which is
|
||||
// derived at write time rather than stored.
|
||||
imagePipeline: string
|
||||
imagePipelineParams: Record<string, unknown>
|
||||
imageVectorEnabled: boolean
|
||||
imageActions: ImageActionsConfig
|
||||
asrConfig: { enabled: boolean; modelId: string; language: string }
|
||||
|
|
@ -1231,7 +1250,8 @@ function createDefaultUIState(): UploadUIState {
|
|||
tableMetadataInstructions: '',
|
||||
},
|
||||
multimodalConfig: { enabled: false, vllmModelId: '', descriptionLanguage: '', customInstructions: '' },
|
||||
imageAttrsEnabled: false,
|
||||
imagePipeline: IMAGE_PIPELINE_DEFAULT,
|
||||
imagePipelineParams: {},
|
||||
imageVectorEnabled: false,
|
||||
imageActions: mergeImageActions(),
|
||||
asrConfig: { enabled: false, modelId: '', language: '' },
|
||||
|
|
@ -1276,8 +1296,11 @@ function initFromKbInfo(kb: any) {
|
|||
descriptionLanguage: kb.vlm_config?.description_language || '',
|
||||
customInstructions: kb.vlm_config?.custom_instructions || '',
|
||||
},
|
||||
// 默认跟随知识库的图片属性观察设置;用户可对本次任务单独覆盖。
|
||||
imageAttrsEnabled: !!kb.image_processing_config?.image_attrs_enabled,
|
||||
// 默认跟随知识库的图片解析方案;用户可对本次任务单独覆盖。旧库(没有
|
||||
// image_pipeline 的)按与后端一致的规则解析出一个具体方案。
|
||||
imagePipeline: resolveImagePipelineFromKb(kb),
|
||||
imagePipelineParams:
|
||||
(kb.image_processing_config?.image_pipeline_params as Record<string, unknown>) ?? {},
|
||||
imageVectorEnabled: !!kb.image_processing_config?.image_vector_enabled,
|
||||
imageActions: mergeImageActions(kb.image_processing_config?.image_actions),
|
||||
asrConfig: {
|
||||
|
|
@ -1328,7 +1351,9 @@ function buildProcessOverrides(): KnowledgeProcessOverrides {
|
|||
table_metadata_instructions: chunking.tableMetadataInstructions,
|
||||
},
|
||||
enable_multimodel: state.multimodalConfig.enabled,
|
||||
image_attrs_enabled: state.imageAttrsEnabled,
|
||||
// 旧观察开关跟随选中的方案(只有智能模式点亮它),与后端
|
||||
// EffectiveProcessConfig 的派生规则一致;方案的显式选择随行下发。
|
||||
image_attrs_enabled: state.imagePipeline === IMAGE_PIPELINE_SMARTOCR,
|
||||
image_vector_enabled: state.imageVectorEnabled,
|
||||
image_actions: {
|
||||
ocr: {
|
||||
|
|
@ -1365,6 +1390,20 @@ function buildProcessOverrides(): KnowledgeProcessOverrides {
|
|||
},
|
||||
}
|
||||
|
||||
// 方案的显式选择与私有参数由共享助手产出:没选方案、没有可调参数、或多模态
|
||||
// 关着(此时本节的控件整个不显示,用户没得选)时整段不下发。
|
||||
//
|
||||
// 多模态关着时必须不下发:后端 EffectiveProcessConfig 一见到 image_pipeline
|
||||
// 就会用方案 id 反推 ImageAttrsEnabled,那会把这里显式发的 false 顶回去,
|
||||
// 于是「关掉多模态」对属性观察失去作用。这与知识库编辑器的写法规律一致。
|
||||
Object.assign(
|
||||
overrides,
|
||||
buildPipelineFields(
|
||||
state.multimodalConfig.enabled ? state.imagePipeline : '',
|
||||
state.imagePipelineParams,
|
||||
),
|
||||
)
|
||||
|
||||
if (state.pdfForceScanned) {
|
||||
overrides.parser_engine_overrides = {
|
||||
pdf_force_scanned: 'true',
|
||||
|
|
@ -1394,7 +1433,17 @@ function applyOverridesToState(o?: KnowledgeProcessOverrides | null) {
|
|||
}
|
||||
if (o.parser_engine_rules) s.chunkingConfig.parserEngineRules = o.parser_engine_rules
|
||||
if (o.enable_multimodel != null) s.multimodalConfig.enabled = o.enable_multimodel
|
||||
if (o.image_attrs_enabled != null) s.imageAttrsEnabled = o.image_attrs_enabled
|
||||
if (o.image_pipeline != null) {
|
||||
// 存量覆盖里可能是更名前的 caption_ocr / ob_cap_ocr,交给共享助手归一。
|
||||
s.imagePipeline = normalizeImagePipelineId(o.image_pipeline) || IMAGE_PIPELINE_DEFAULT
|
||||
// 与后端 ResolveProcessConfig 一致:显式方案和参数整体替换,缺省参数使用方案默认值。
|
||||
s.imagePipelineParams = o.image_pipeline_params ?? {}
|
||||
} else if (o.image_attrs_enabled != null) {
|
||||
// 旧版观察开关只在切换方案时丢弃原参数;同一方案继续继承知识库设置。
|
||||
const pipeline = o.image_attrs_enabled ? IMAGE_PIPELINE_SMARTOCR : IMAGE_PIPELINE_DEFAULT
|
||||
if (pipeline !== s.imagePipeline) s.imagePipelineParams = {}
|
||||
s.imagePipeline = pipeline
|
||||
}
|
||||
if (o.image_vector_enabled != null) s.imageVectorEnabled = o.image_vector_enabled
|
||||
if (o.image_actions) s.imageActions = mergeImageActions(o.image_actions)
|
||||
if (o.vlm_config) {
|
||||
|
|
@ -1597,12 +1646,19 @@ const handleCancel = () => {
|
|||
emit('update:visible', false)
|
||||
}
|
||||
|
||||
// 面板报上来的「选中的方案一个动作都没开」;多模态开启时阻止确认。
|
||||
const imagePipelineInvalid = ref(false)
|
||||
|
||||
const handleConfirm = () => {
|
||||
if (props.mode === 'file' && batchItemCount.value === 0) {
|
||||
MessagePlugin.warning(t('uploadConfirm.noItems'))
|
||||
return
|
||||
}
|
||||
if (!validateBeforeConfirm()) return
|
||||
if (uiState.value.multimodalConfig.enabled && imagePipelineInvalid.value) {
|
||||
MessagePlugin.warning(t('imagePipeline.errors.noActionEnabled'))
|
||||
return
|
||||
}
|
||||
|
||||
const processConfig = buildProcessOverrides()
|
||||
if (props.mode === 'manual' && props.manualPreview) {
|
||||
|
|
@ -2373,6 +2429,9 @@ const handleConfirm = () => {
|
|||
|
||||
.setting-row-vertical {
|
||||
flex-direction: column;
|
||||
// stretch 而非继承的 flex-start:竖排行的子块(如图片解析方案面板)必须
|
||||
// 占满整行,内部控件才能与其它行的开关一样靠右对齐(与知识库编辑器一致)。
|
||||
align-items: stretch;
|
||||
gap: 12px;
|
||||
|
||||
.setting-info,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import test from 'node:test'
|
|||
import { compileScript, parse } from '@vue/compiler-sfc'
|
||||
import ts from 'typescript'
|
||||
import { createRenderer, h } from 'vue'
|
||||
import * as imageProcessingConfig from '../../../utils/imageProcessingConfig'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const filename = fileURLToPath(new URL('./UploadConfirmDialog.vue', import.meta.url))
|
||||
|
|
@ -29,6 +30,7 @@ function fixture(kbInfo: any) {
|
|||
if (name === '@/stores/editorResources') return { useEditorResourcesStore: () => ({}) }
|
||||
if (name === '@/stores/ui') return { useUIStore: () => ({}) }
|
||||
if (name === '@/api/knowledge-base') return { mergeImageActions: () => ({ ocr: { on: [], on_unobserved: true } }) }
|
||||
if (name === '@/utils/imageProcessingConfig') return imageProcessingConfig
|
||||
return { default: {} }
|
||||
},
|
||||
})
|
||||
|
|
@ -88,3 +90,55 @@ test('image capability warning uses the KB embedding model and tolerates shared
|
|||
assert.equal(f.vm.embeddingTakesImages, true)
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
||||
for (const pipelineId of ['default', 'smartocr', 'caption_ocr', 'ob_cap_ocr']) {
|
||||
for (const params of [undefined, {}, { 'caption.thinking': false }]) {
|
||||
test(`reparse replaces KB parameters with document ${pipelineId} parameters ${JSON.stringify(params)}`, () => {
|
||||
const kb = {
|
||||
vlm_config: { enabled: true, model_id: 'vlm' },
|
||||
image_processing_config: { image_pipeline: 'default', image_pipeline_params: { enable_caption: false, 'caption.thinking': true } },
|
||||
}
|
||||
const f = fixture(kb)
|
||||
try {
|
||||
f.vm.applyOverridesToState({ image_pipeline: pipelineId, image_pipeline_params: params })
|
||||
const result = JSON.parse(JSON.stringify(f.vm.buildProcessOverrides()))
|
||||
assert.equal(result.image_pipeline, imageProcessingConfig.normalizeImagePipelineId(pipelineId))
|
||||
assert.deepEqual(result.image_pipeline_params ?? {}, params ?? {})
|
||||
assert.deepEqual(kb.image_processing_config.image_pipeline_params, { enable_caption: false, 'caption.thinking': true })
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const kbPipeline of ['default', 'smartocr']) {
|
||||
for (const attrsEnabled of [false, true]) {
|
||||
test(`legacy reparse override ${attrsEnabled} only retains parameters from ${kbPipeline} when the pipeline matches`, () => {
|
||||
const params = { 'caption.thinking': true }
|
||||
const f = fixture({
|
||||
vlm_config: { enabled: true, model_id: 'vlm' },
|
||||
image_processing_config: { image_pipeline: kbPipeline, image_pipeline_params: params },
|
||||
})
|
||||
try {
|
||||
f.vm.applyOverridesToState({ image_attrs_enabled: attrsEnabled })
|
||||
const result = JSON.parse(JSON.stringify(f.vm.buildProcessOverrides()))
|
||||
const pipeline = attrsEnabled ? 'smartocr' : 'default'
|
||||
assert.equal(result.image_pipeline, pipeline)
|
||||
assert.deepEqual(result.image_pipeline_params ?? {}, kbPipeline === pipeline ? params : {})
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
test('reparse inherits KB parameters when the document has no pipeline selection', () => {
|
||||
const params = { enable_caption: false }
|
||||
const f = fixture({
|
||||
vlm_config: { enabled: true, model_id: 'vlm' },
|
||||
image_processing_config: { image_pipeline: 'default', image_pipeline_params: params },
|
||||
})
|
||||
try {
|
||||
for (const overrides of [undefined, { summary_enabled: false }, { image_pipeline_params: { enable_caption: true } }]) {
|
||||
f.vm.applyOverridesToState(overrides)
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(f.vm.buildProcessOverrides().image_pipeline_params)), params)
|
||||
}
|
||||
} finally { f.dispose() }
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,124 +0,0 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ActionRound is one dispatch step: a set of actions executed together. The
|
||||
// dispatch layer may physically merge co-scheduled actions (observation and
|
||||
// caption share one VLM call); the set is what the loop reasons about.
|
||||
type ActionRound struct {
|
||||
Actions []types.ActionKind
|
||||
}
|
||||
|
||||
// Contains reports whether the round schedules the given action.
|
||||
func (r ActionRound) Contains(k types.ActionKind) bool {
|
||||
for _, a := range r.Actions {
|
||||
if a == k {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// selectImageActionRounds turns the payload switches into the initial plan.
|
||||
// Observation mode opens with an observation+caption round and lets the result
|
||||
// processor append an OCR round; caption+OCR mode schedules caption and OCR as
|
||||
// independent rounds (caption first, then OCR) with no observation. Default
|
||||
// action membership lives in types.DefaultEnabledActions; the whole-task
|
||||
// switches (EnableCaption/EnableOCR) only gate caption and OCR on top of it.
|
||||
func selectImageActionRounds(p *types.ImageMultimodalPayload) []ActionRound {
|
||||
on := types.DefaultEnabledActions(p.ImageAttrsEnabled)
|
||||
if !p.EnableCaption {
|
||||
on[types.ActionCaption] = false
|
||||
}
|
||||
if !p.EnableOCR {
|
||||
on[types.ActionOCR] = false
|
||||
}
|
||||
|
||||
if p.ImageAttrsEnabled {
|
||||
// Observation and caption share one VLM call in attribute mode.
|
||||
r := ActionRound{}
|
||||
if on[types.ActionCaption] {
|
||||
r.Actions = append(r.Actions, types.ActionCaption)
|
||||
}
|
||||
if on[types.ActionObservation] {
|
||||
r.Actions = append(r.Actions, types.ActionObservation)
|
||||
}
|
||||
if len(r.Actions) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []ActionRound{r}
|
||||
}
|
||||
|
||||
var rounds []ActionRound
|
||||
if on[types.ActionCaption] {
|
||||
rounds = append(rounds, ActionRound{Actions: []types.ActionKind{types.ActionCaption}})
|
||||
}
|
||||
if on[types.ActionOCR] {
|
||||
rounds = append(rounds, ActionRound{Actions: []types.ActionKind{types.ActionOCR}})
|
||||
}
|
||||
return rounds
|
||||
}
|
||||
|
||||
// executeActionRound runs one dispatch step. Observation and caption are
|
||||
// physically merged into a single VLM call (the describe round); a caption-only
|
||||
// round (caption+OCR mode) uses the plain caption prompt; an OCR-only round uses
|
||||
// prompt. The transport here is the direct vlmModel.Predict shim — PR-E will
|
||||
// route it through a VLM request manager without touching this dispatch logic.
|
||||
func (s *ImageMultimodalService) executeActionRound(
|
||||
ctx context.Context,
|
||||
payload *types.ImageMultimodalPayload,
|
||||
model vlm.VLM,
|
||||
imgBytes []byte,
|
||||
vlmCfg types.VLMConfig,
|
||||
imageInfo *types.ImageInfo,
|
||||
out types.JSONMap,
|
||||
round ActionRound,
|
||||
) {
|
||||
if round.Contains(types.ActionObservation) {
|
||||
raw, capErr := model.Predict(ctx, [][]byte{imgBytes}, buildImageAttrsPrompt(ctx, vlmCfg))
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Describe and observe failed for %s: %v", payload.ImageURL, capErr)
|
||||
out["caption_error"] = capErr.Error()
|
||||
return
|
||||
}
|
||||
obs, ok := types.ParseImageAttrsResponse(raw)
|
||||
applyImageObservation(imageInfo, obs.Attrs, obs.Description, out, payload.EnableCaption)
|
||||
if !obs.Observed {
|
||||
// The model ignored the attribute protocol — a user custom
|
||||
// instruction may have derailed the format, or it answered in prose.
|
||||
// Nothing is invented to fill the gap: the attribute table stays
|
||||
// empty and the OCR policy falls back to its conservative
|
||||
// OnUnobserved clause. Flag it so the trace shows the observation
|
||||
// did not actually happen, instead of the empty table being read as
|
||||
// a confident "nothing to see here".
|
||||
out["observation_failed"] = true
|
||||
}
|
||||
if !ok {
|
||||
out["caption_missing"] = true
|
||||
}
|
||||
return
|
||||
}
|
||||
if round.Contains(types.ActionCaption) {
|
||||
raw, capErr := model.Predict(ctx, [][]byte{imgBytes}, buildVLMCaptionPrompt(ctx, vlmCfg))
|
||||
if capErr != nil {
|
||||
out["caption_error"] = capErr.Error()
|
||||
return
|
||||
}
|
||||
if text := strings.TrimSpace(raw); text != "" {
|
||||
imageInfo.Caption = text
|
||||
out["caption_chars"] = len([]rune(text))
|
||||
out["caption_preview"] = previewText(text, 200)
|
||||
}
|
||||
return
|
||||
}
|
||||
if round.Contains(types.ActionOCR) {
|
||||
s.runImageOCR(ctx, payload, model, imgBytes, imageInfo, out, vlmCfg)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,586 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// imagePipeline is one image pipeline: a function over actions, not a data
|
||||
// topology. Run owns the pipeline's whole control flow, which is why there is no
|
||||
// scheduler and no round model in this package — see the image pipeline spec §2.
|
||||
type imagePipeline interface {
|
||||
// ID is the stable id of the pipeline; it doubles as the value recorded on
|
||||
// the trace row.
|
||||
ID() types.ImagePipelineID
|
||||
// Name is the human-readable name shown in the settings panel.
|
||||
Name() string
|
||||
// Description tells the user how the pipeline works, shown under the
|
||||
// pick in the settings panel.
|
||||
Description() string
|
||||
// Fields are the tunables of this pipeline; nil means it has none yet.
|
||||
Fields() []types.ImageFieldDef
|
||||
// Rules are the conditions those tunables have to meet. Declared rather
|
||||
// than implemented because what makes a set of settings incoherent is a
|
||||
// fact about the pipeline, not about the panel that renders it: the manual
|
||||
// pipeline can have every action switched off, the smart one schedules its
|
||||
// own work and cannot be. The panel asks the backend anyway — the save
|
||||
// path re-checks what the panel approved — so nothing here has to trust it.
|
||||
Rules() []types.ImagePipelineRules
|
||||
// Validate reports whether a saved set of tunables can be run. Returning
|
||||
// nil means the pipeline accepts anything handed to it.
|
||||
Validate(params map[string]any) error
|
||||
// Run executes the pipeline.
|
||||
Run(ctx context.Context, r *runContext) error
|
||||
}
|
||||
|
||||
var imagePipelineRegistry = make(map[types.ImagePipelineID]imagePipeline)
|
||||
|
||||
func registerImagePipeline(p imagePipeline) {
|
||||
if _, dup := imagePipelineRegistry[p.ID()]; dup {
|
||||
// init() order inside a package is file-name order, so a silent
|
||||
// overwrite would sit here until someone noticed the wrong pipeline ran.
|
||||
panic(fmt.Sprintf("image pipeline: duplicate pipeline id %q", p.ID()))
|
||||
}
|
||||
imagePipelineRegistry[p.ID()] = p
|
||||
}
|
||||
|
||||
// selectImagePipeline resolves the pipeline a task asked for. Both pipelines
|
||||
// register from their own file's init(). An empty or unregistered id falls back
|
||||
// to the attribute-observation switch: that is what a payload enqueued before
|
||||
// the pipeline field existed carries, and what a knowledge base saved before the
|
||||
// selector existed asks for. Falling back rather than failing keeps a stored id
|
||||
// that this build no longer ships from stranding its images.
|
||||
func selectImagePipeline(payload *types.ImageMultimodalPayload) imagePipeline {
|
||||
id := types.ImagePipelineID(strings.TrimSpace(string(payload.ImagePipelineID)))
|
||||
return resolveImagePipeline(id, payload.ImageAttrsEnabled)
|
||||
}
|
||||
|
||||
// resolveImagePipeline is selectImagePipeline's lookup, separated so the
|
||||
// validation entry point can ask the same question about a pipeline that has
|
||||
// not been turned into a payload yet.
|
||||
func resolveImagePipeline(pipelineID types.ImagePipelineID, attrsEnabled bool) imagePipeline {
|
||||
fallback := types.ImagePipelineIDFor(attrsEnabled)
|
||||
// A stored id may name a pipeline by a spelling this build renamed; the
|
||||
// normalization keeps such a config on its successor instead of dropping
|
||||
// to the fallback silently.
|
||||
id := types.NormalizeImagePipelineID(pipelineID)
|
||||
if id == "" {
|
||||
id = fallback
|
||||
}
|
||||
if p, ok := imagePipelineRegistry[id]; ok {
|
||||
return p
|
||||
}
|
||||
return imagePipelineRegistry[fallback]
|
||||
}
|
||||
|
||||
// ValidateImagePipelineParams asks the pipeline an id names whether the
|
||||
// tunables a knowledge base is about to store can be run. The id is resolved
|
||||
// exactly as a running task would resolve it, so the answer cannot disagree
|
||||
// with the pipeline that would actually handle the images.
|
||||
func ValidateImagePipelineParams(pipelineID types.ImagePipelineID, params map[string]any) error {
|
||||
return firstImagePipelineError(BrokenImagePipelineRules(pipelineID, params))
|
||||
}
|
||||
|
||||
// BrokenImagePipelineRules lists every rule of the named pipeline that the
|
||||
// stored tunables break. Unlike the single error ValidateImagePipelineParams
|
||||
// returns, this keeps each violation separate so that the settings panel can
|
||||
// show all of them at once — one rule per line, each pointing at the control
|
||||
// that has to change. A pipeline that declares no rules yields none, which is
|
||||
// how a pipeline whose actions run by themselves is never told it is wrong.
|
||||
func BrokenImagePipelineRules(
|
||||
pipelineID types.ImagePipelineID,
|
||||
params map[string]any,
|
||||
) []*types.ImagePipelineValidationError {
|
||||
pipeline := resolveImagePipeline(pipelineID, false)
|
||||
if pipeline == nil {
|
||||
return nil
|
||||
}
|
||||
return EvaluateImagePipelineRules(pipeline.ID(), pipeline.Rules(), pipeline.Fields(), params)
|
||||
}
|
||||
|
||||
// ListImagePipelines renders every registered pipeline as the settings panel
|
||||
// sees it. Sorted by id so the panel's option order is stable and does not
|
||||
// follow package init order.
|
||||
func ListImagePipelines() []types.ImagePipelineSpec {
|
||||
ids := make([]types.ImagePipelineID, 0, len(imagePipelineRegistry))
|
||||
for id := range imagePipelineRegistry {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||||
|
||||
specs := make([]types.ImagePipelineSpec, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
p := imagePipelineRegistry[id]
|
||||
specs = append(specs, types.ImagePipelineSpec{
|
||||
ID: p.ID(),
|
||||
Name: p.Name(),
|
||||
Description: p.Description(),
|
||||
Fields: p.Fields(),
|
||||
Rules: p.Rules(),
|
||||
})
|
||||
}
|
||||
return specs
|
||||
}
|
||||
|
||||
// actionHandler is the executable half of an image action. There is deliberately
|
||||
// no service receiver: every current action is a pure function of the run
|
||||
// context, so running one needs no wiring.
|
||||
type actionHandler func(ctx context.Context, r *runContext) error
|
||||
|
||||
// actionSpec joins the declarative half declared in types with its handler.
|
||||
type actionSpec struct {
|
||||
types.ImageActionSpec
|
||||
handler actionHandler
|
||||
}
|
||||
|
||||
var imageActionRegistry = make(map[types.ImageActionID]actionSpec)
|
||||
|
||||
func registerAction(spec actionSpec) {
|
||||
if _, dup := imageActionRegistry[spec.ID]; dup {
|
||||
panic(fmt.Sprintf("image pipeline: duplicate action id %q", spec.ID))
|
||||
}
|
||||
imageActionRegistry[spec.ID] = spec
|
||||
}
|
||||
|
||||
// init binds the shared actions to their handlers. The bindings live here rather
|
||||
// than next to the declarations so that "every action is registered in exactly
|
||||
// one place" stays true as pipelines grow.
|
||||
func init() {
|
||||
handlers := map[types.ImageActionID]actionHandler{
|
||||
types.ImageActionCaption: runCaptionAction,
|
||||
types.ImageActionObservationCaption: runObservationCaptionAction,
|
||||
types.ImageActionOCR: runOCRAction,
|
||||
}
|
||||
for _, spec := range types.SharedImageActions {
|
||||
handler, bound := handlers[spec.ID]
|
||||
if !bound {
|
||||
panic(fmt.Sprintf("image pipeline: no handler bound to shared action %q", spec.ID))
|
||||
}
|
||||
registerAction(actionSpec{ImageActionSpec: spec, handler: handler})
|
||||
}
|
||||
}
|
||||
|
||||
// Thinking-switch keys of the shared actions, in the action's own namespace
|
||||
// ("action.key"). An action may want the model to reason — a dense scanned
|
||||
// page or a chart that has to be read before it can be transcribed — and must
|
||||
// not pay for it on the simple majority, so the switch belongs to the ACTION,
|
||||
// not to the model and not to any pipeline: whichever pipeline schedules the
|
||||
// caption or the OCR, the tunable means the same thing and reads the same
|
||||
// stored key. Pipelines declare these same keys in their Fields() so the
|
||||
// panel renders them; the keys never collide with pipeline-private ones
|
||||
// because the action namespace is distinct.
|
||||
const (
|
||||
imageFieldKeyCaptionThinking = "caption.thinking"
|
||||
imageFieldKeyOCRThinking = "ocr.thinking"
|
||||
)
|
||||
|
||||
// imageActionThinkingFields are the thinking switches DECLARED ONCE here, at
|
||||
// the action layer, and carried into every pipeline's Fields() by reference —
|
||||
// a pipeline does not re-declare or re-name an action's tunable, it only
|
||||
// offers it on its panel. The keys are namespaced by action ("action.key"),
|
||||
// so the same tunable reads the same stored value whichever pipeline
|
||||
// schedules the action.
|
||||
var imageActionThinkingFields = []types.ImageFieldDef{
|
||||
{
|
||||
Key: imageFieldKeyCaptionThinking,
|
||||
Type: types.ImageFieldTypeBool,
|
||||
Label: "Caption with thinking",
|
||||
// Warning, not recommendation: on some models long reasoning can
|
||||
// crowd out the answer itself and leave the caption empty.
|
||||
Description: "Let the model think before writing the caption. " +
|
||||
"Rarely needed: it costs latency, and on some models long " +
|
||||
"reasoning truncates the caption. Enable only when necessary.",
|
||||
Default: false,
|
||||
// No DecidesAction: an image is still captioned with thinking off,
|
||||
// just faster and with less care, so this switch cannot be the one
|
||||
// that leaves the pipeline with nothing to do.
|
||||
},
|
||||
{
|
||||
Key: imageFieldKeyOCRThinking,
|
||||
Type: types.ImageFieldTypeBool,
|
||||
Label: "OCR with thinking",
|
||||
Description: "Let the model think before transcribing. " +
|
||||
"Rarely needed: it costs latency, and on some models long " +
|
||||
"reasoning truncates the text. Enable only when necessary.",
|
||||
Default: false,
|
||||
// Same reasoning: an image with no text still yields no OCR.
|
||||
},
|
||||
}
|
||||
|
||||
// runContext is the only framework entry point a pipeline or an action gets. It
|
||||
// hands out no VLM handle: talking to a model is the action's business, reached
|
||||
// only through execute.
|
||||
type runContext struct {
|
||||
payload *types.ImageMultimodalPayload
|
||||
model vlm.VLM
|
||||
imageBytes []byte
|
||||
vlmCfg types.VLMConfig
|
||||
imageInfo *types.ImageInfo
|
||||
out types.JSONMap
|
||||
// pipelineID is the pipeline serving this run, used to namespace
|
||||
// pipeline-private param lookups ("<pipeline>.<key>"). Empty in tests
|
||||
// that build a runContext by hand, which then read bare keys only.
|
||||
pipelineID string
|
||||
// params is this pipeline's private tunables, as resolved from the
|
||||
// knowledge base. Nothing else may read it: a key understood by one
|
||||
// pipeline says nothing about another's, so the key only has to be
|
||||
// unambiguous within the pipeline that declared it.
|
||||
params map[string]any
|
||||
// declared is the pipeline's own imagePipeline.Fields(), carried along so
|
||||
// an untouched key resolves to the default the panel promised rather than
|
||||
// to a zero value. It is the same list the frontend renders.
|
||||
declared []types.ImageFieldDef
|
||||
// actions records the actions this run actually executed, in order. It is
|
||||
// the per-image answer to "what did this image go through", which a trace
|
||||
// row's pipeline label alone cannot tell.
|
||||
actions []types.ImageActionID
|
||||
// The tunable that switches thinking on for each action. A pipeline names
|
||||
// its controls after its own vocabulary — the manual pipeline speaks of a
|
||||
// caption, the observing one of a description — and no key may be declared
|
||||
// by two pipelines, so it is the pipeline, not the action, that says which
|
||||
// tunable a thinking switch is. Filled in by the pipeline before it
|
||||
// executes anything.
|
||||
captionThinkingKey string
|
||||
ocrThinkingKey string
|
||||
// tracker and imgSpan carry the per-image trace plumbing so the OCR
|
||||
// action can raise and resolve its own subspan ("multimodal.image[n].ocr")
|
||||
// at the exact spot it runs — a failed OCR fails that subspan and the
|
||||
// outcome summary without failing or retrying the whole image. Both are
|
||||
// nil when the attempt has no parent span to hang a subspan on.
|
||||
tracker SpanTracker
|
||||
imgSpan *Span
|
||||
// vlmErr keeps the FIRST typed VLM error the run recorded. Actions
|
||||
// record failures instead of propagating them (#4132: a failed OCR must
|
||||
// not fail an image that already produced a valid caption), but when the
|
||||
// run ends with nothing produced at all, the task layer needs the typed
|
||||
// error to tell a permanent failure (SkipRetry) from a transient one.
|
||||
vlmErr error
|
||||
}
|
||||
|
||||
// Param reads one private tunable, falling back to the default the pipeline
|
||||
// declared for it. A key the running pipeline never declared reads nil, which
|
||||
// is deliberate: a stored knowledge base may carry parameters for a pipeline
|
||||
// this build no longer has, and the run must not fail over a stale key.
|
||||
//
|
||||
// Pipeline-private keys are looked up under the running pipeline's namespace
|
||||
// first ("<pipeline>.<key>"), so two pipelines may one day declare the same
|
||||
// field key without one reading the other's stored value; a bare key still
|
||||
// answers, which keeps stored configs written before namespacing readable.
|
||||
// Action-level keys ("action.key") are stored bare and hit the exact lookup.
|
||||
func (r *runContext) Param(key string) any {
|
||||
if r.params != nil {
|
||||
if r.pipelineID != "" {
|
||||
if v, ok := r.params[r.pipelineID+"."+key]; ok && v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
if v, ok := r.params[key]; ok && v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
for _, field := range r.declared {
|
||||
if field.Key == key {
|
||||
return field.Default
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BoolParam reads a boolean tunable. JSON hands a map[string]any an
|
||||
// unmarshalled "true" as a real bool, but a config written by hand may use a
|
||||
// string, so both are read; anything else falls back to false.
|
||||
func (r *runContext) BoolParam(key string) bool {
|
||||
return r.BoolParamOr(key, false)
|
||||
}
|
||||
|
||||
// BoolParamOr reads a boolean tunable the way BoolParam does, but falls back
|
||||
// to def when neither the stored params nor a declared field default answers.
|
||||
// A pipeline that reads a key it does not declare in Fields() — a control it
|
||||
// deliberately keeps off the panel — pins its working default here, at the
|
||||
// one place the key is read.
|
||||
func (r *runContext) BoolParamOr(key string, def bool) bool {
|
||||
switch v := r.Param(key).(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return v == "true" || v == "1" || v == "on"
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// ParamSnapshot renders the given keys as a plain map for the trace row.
|
||||
func (r *runContext) ParamSnapshot(keys ...string) types.JSONMap {
|
||||
snapshot := make(types.JSONMap, len(keys))
|
||||
for _, key := range keys {
|
||||
if _, declared := r.fieldByKey(key); declared {
|
||||
snapshot[key] = r.Param(key)
|
||||
}
|
||||
}
|
||||
if len(snapshot) == 0 {
|
||||
return nil
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// predictCaption asks for a caption, with the tunable the current pipeline
|
||||
// offers for it. purpose labels the underlying VLM call in the usage log so the
|
||||
// observation step can be told apart from OCR after the fact: a plain caption
|
||||
// passes "image_caption", the attribute-observing caption passes
|
||||
// "image_observation".
|
||||
func (r *runContext) predictCaption(ctx context.Context, prompt, purpose string) (string, error) {
|
||||
return r.think(types.WithLLMCallMetadata(ctx, purpose, ""), prompt,
|
||||
r.captionThinkingKey, imageFieldKeyCaptionThinking)
|
||||
}
|
||||
|
||||
// predictOCR transcribes text from an image, with the tunable the current
|
||||
// pipeline offers for it. The image_ocr purpose labels the VLM call in the
|
||||
// usage log so it can be told apart from the observation step.
|
||||
func (r *runContext) predictOCR(ctx context.Context, prompt string) (string, error) {
|
||||
return r.think(types.WithLLMCallMetadata(ctx, "image_ocr", ""), prompt,
|
||||
r.ocrThinkingKey, imageFieldKeyOCRThinking)
|
||||
}
|
||||
|
||||
// think asks the model with the thinking switch the pipeline wired to this
|
||||
// action. Thinking defaults to off because on a quantized reasoning model the
|
||||
// reasoning spends the completion budget before a single visible token, and the
|
||||
// caller cannot tell that empty answer from an image that genuinely carries no
|
||||
// text — which is exactly how an OCR result comes to be discarded as invalid.
|
||||
// Turning it on costs a longer run and is what a dense or degraded image wants,
|
||||
// so the switch is per action. The effective value reaches the trace under the
|
||||
// SHARED action-level key, never under the pipeline's own field key: pipelines
|
||||
// own private param keys (per-pipeline storage isolation — one pipeline's
|
||||
// stored switch must not drive another's run), but a run's trace names the
|
||||
// same switch the same way whichever pipeline served it.
|
||||
func (r *runContext) think(ctx context.Context, prompt, paramKey, traceKey string) (string, error) {
|
||||
on := r.BoolParamOr(paramKey, false)
|
||||
r.out[traceKey] = on
|
||||
return r.model.PredictWithOptions(ctx, [][]byte{r.imageBytes}, prompt,
|
||||
&vlm.PredictOptions{Thinking: &on})
|
||||
}
|
||||
|
||||
func (r *runContext) fieldByKey(key string) (types.ImageFieldDef, bool) {
|
||||
for _, field := range r.declared {
|
||||
if field.Key == key {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
return types.ImageFieldDef{}, false
|
||||
}
|
||||
|
||||
// execute runs one action by id. A missing or unbound action costs that one
|
||||
// field, not the whole image: interrupting would drop every later slot of the
|
||||
// same image for the sake of a warning.
|
||||
func (r *runContext) execute(ctx context.Context, id types.ImageActionID) error {
|
||||
spec, ok := imageActionRegistry[id]
|
||||
if !ok || spec.handler == nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Action %q is not implemented; skipping", id)
|
||||
r.out["action_skipped"] = string(id)
|
||||
return nil
|
||||
}
|
||||
r.actions = append(r.actions, id)
|
||||
return spec.handler(ctx, r)
|
||||
}
|
||||
|
||||
// runCaptionAction asks for a one-line description and stores it as the caption.
|
||||
// Its failure subspan mirrors the OCR one (#4132): a caption miss is recorded
|
||||
// (caption_error + a red .caption subspan) without failing the image.
|
||||
func runCaptionAction(ctx context.Context, r *runContext) error {
|
||||
var captionSpan *Span
|
||||
if r.tracker != nil && r.imgSpan != nil {
|
||||
captionSpan = r.tracker.BeginSubSpan(ctx, r.imgSpan, r.imgSpan.Name+".caption",
|
||||
types.SpanKindGeneration, nil)
|
||||
}
|
||||
resolve := func(failed bool) {
|
||||
if captionSpan == nil {
|
||||
return
|
||||
}
|
||||
if failed {
|
||||
message, _ := r.out["caption_error"].(string)
|
||||
r.tracker.FailSpan(ctx, captionSpan, "CAPTION_FAILED", message, nil)
|
||||
return
|
||||
}
|
||||
r.tracker.EndSpan(ctx, captionSpan, types.JSONMap{
|
||||
"status": "succeeded", "chars": r.out["caption_chars"],
|
||||
})
|
||||
}
|
||||
raw, err := r.predictCaption(ctx, buildVLMCaptionPrompt(ctx, r.vlmCfg), "image_caption")
|
||||
if err != nil {
|
||||
// Only recorded, not logged: an observation failure below is the same
|
||||
// kind of event and logs its own line, and a caption miss must not be
|
||||
// louder than the rest of the run.
|
||||
r.out["caption_error"] = err.Error()
|
||||
if r.vlmErr == nil {
|
||||
r.vlmErr = err
|
||||
}
|
||||
resolve(true)
|
||||
return nil
|
||||
}
|
||||
if text := strings.TrimSpace(raw); text != "" {
|
||||
r.imageInfo.Caption = text
|
||||
r.out["caption_chars"] = len([]rune(text))
|
||||
r.out["caption_preview"] = previewText(text, 200)
|
||||
}
|
||||
resolve(false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runObservationCaptionAction observes the registered attributes and describes
|
||||
// the image in one answer. The description reaches the caption slot only when the
|
||||
// run wants a caption — the observation itself always happens, because the
|
||||
// attributes are the point of this action and the OCR policy reads them back.
|
||||
func runObservationCaptionAction(ctx context.Context, r *runContext) error {
|
||||
var obsSpan *Span
|
||||
if r.tracker != nil && r.imgSpan != nil {
|
||||
obsSpan = r.tracker.BeginSubSpan(ctx, r.imgSpan, r.imgSpan.Name+".observe_caption",
|
||||
types.SpanKindGeneration, nil)
|
||||
}
|
||||
resolve := func(failed bool) {
|
||||
if obsSpan == nil {
|
||||
return
|
||||
}
|
||||
if failed {
|
||||
message, _ := r.out["caption_error"].(string)
|
||||
r.tracker.FailSpan(ctx, obsSpan, "OBSERVATION_FAILED", message, nil)
|
||||
return
|
||||
}
|
||||
r.tracker.EndSpan(ctx, obsSpan, types.JSONMap{"status": "succeeded"})
|
||||
}
|
||||
raw, err := r.predictCaption(ctx, buildImageAttrsPrompt(ctx, r.vlmCfg), "image_observation")
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Describe and observe failed for %s: %v", r.payload.ImageURL, err)
|
||||
r.out["caption_error"] = err.Error()
|
||||
// The request itself failed: no attributes will ever arrive. Flag it so
|
||||
// the pipeline's OCR decision can tell "observed and declined" from
|
||||
// "no observation to decide on".
|
||||
r.out["observation_failed"] = true
|
||||
if r.vlmErr == nil {
|
||||
r.vlmErr = err
|
||||
}
|
||||
resolve(true)
|
||||
return nil
|
||||
}
|
||||
obs, ok := types.ParseImageAttrsResponse(raw)
|
||||
// Whether the description reaches the caption slot is this pipeline's
|
||||
// business, so the observation reads its own key rather than a payload
|
||||
// field. The key is not declared in Fields(), so the working default is
|
||||
// pinned here: with it, the caption stays a by-product; an API caller
|
||||
// passing false observes the attributes and records nothing to caption.
|
||||
applyImageObservation(r.imageInfo, obs.Attrs, obs.Description, r.out,
|
||||
r.BoolParamOr(smartFieldKeyCaptureCaption, true))
|
||||
if !obs.Observed {
|
||||
// The model ignored the attribute protocol — a user custom instruction
|
||||
// may have derailed the format, or it answered in prose. Nothing is
|
||||
// invented to fill the gap: the attribute table stays empty and the
|
||||
// OCR policy falls back to its conservative OnUnobserved clause.
|
||||
r.out["observation_failed"] = true
|
||||
}
|
||||
if !ok {
|
||||
r.out["caption_missing"] = true
|
||||
}
|
||||
resolve(false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runOCRAction extracts the text the image carries. It is a plain function
|
||||
// rather than a method because the OCR prompt is system-owned (knowledge base
|
||||
// custom instructions must never reach it) and no service state is involved.
|
||||
//
|
||||
// Failure semantics (the #4132 contract): an OCR failure is RECORDED — status,
|
||||
// a stable error code, the raw message, and a failed ".ocr" subspan — but it
|
||||
// does not fail the image. Retrying the whole image would re-run a caption
|
||||
// that already succeeded and duplicate its chunk, so the outcome summary (not
|
||||
// the task retry) is what surfaces the failure.
|
||||
func runOCRAction(ctx context.Context, r *runContext) error {
|
||||
// The OCR subspan rides on the image span; without a parent span there is
|
||||
// nothing to hang it on and the outcome fields carry the signal alone.
|
||||
var ocrSpan *Span
|
||||
if r.tracker != nil && r.imgSpan != nil {
|
||||
ocrSpan = r.tracker.BeginSubSpan(ctx, r.imgSpan, r.imgSpan.Name+".ocr",
|
||||
types.SpanKindGeneration, nil)
|
||||
}
|
||||
resolve := func() {
|
||||
if ocrSpan == nil {
|
||||
return
|
||||
}
|
||||
// The code reads back from out so the span always matches what the
|
||||
// outcome summary sees — the caller may have refined it (e.g. a
|
||||
// request failure re-coded as OCR_TRUNCATED) after the initial set.
|
||||
if code, _ := r.out["ocr_error_code"].(string); code != "" {
|
||||
message, _ := r.out["ocr_error"].(string)
|
||||
r.tracker.FailSpan(ctx, ocrSpan, code, message, nil)
|
||||
return
|
||||
}
|
||||
r.tracker.EndSpan(ctx, ocrSpan, types.JSONMap{
|
||||
"status": r.out["ocr_status"], "chars": r.out["ocr_chars"],
|
||||
})
|
||||
}
|
||||
// A failed extraction must never leave text from a previous attempt behind.
|
||||
r.imageInfo.OCRText = ""
|
||||
r.out["ocr_chars"] = 0
|
||||
|
||||
// The OCR prompt is system-owned: knowledge base custom instructions must
|
||||
// never reach it, or free-form business rules compete with the "No text
|
||||
// content" contract and poison image_ocr chunks. buildVLMOCRPrompt picks
|
||||
// the scanned-PDF or default prompt and ignores vlmCfg on purpose.
|
||||
prompt := buildVLMOCRPrompt(r.payload.ImageSourceType, r.vlmCfg)
|
||||
if r.payload.ImageSourceType == "scanned_pdf" {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", r.payload.ImageURL)
|
||||
r.out["ocr_prompt"] = "scanned_pdf"
|
||||
} else {
|
||||
r.out["ocr_prompt"] = "default"
|
||||
}
|
||||
|
||||
ocrText, err := r.predictOCR(ctx, prompt)
|
||||
if err != nil {
|
||||
// A model-side failure is not "this image carries no text" (the loss
|
||||
// #4064 describes), but retrying the whole image is not the answer
|
||||
// either — the caption may already be valid. Record the failure with
|
||||
// a stable code and let the outcome summary classify the image.
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", r.payload.ImageURL, err)
|
||||
if r.vlmErr == nil {
|
||||
r.vlmErr = err
|
||||
}
|
||||
r.out["ocr_status"] = "failed"
|
||||
r.out["ocr_error_code"] = "OCR_REQUEST_FAILED"
|
||||
if errors.Is(err, vlm.ErrTruncatedCompletion) {
|
||||
r.out["ocr_error_code"] = "OCR_TRUNCATED"
|
||||
}
|
||||
r.out["ocr_error"] = err.Error()
|
||||
resolve()
|
||||
return nil
|
||||
}
|
||||
r.out["ocr_raw_chars"] = len([]rune(ocrText))
|
||||
ocrText, vErr := validateOCRText(ocrText)
|
||||
if vErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR rejected for %s: %v", r.payload.ImageURL, vErr)
|
||||
r.out["ocr_status"] = "failed"
|
||||
r.out["ocr_error_code"] = "OCR_INVALID_OUTPUT"
|
||||
r.out["ocr_error"] = vErr.Error()
|
||||
var invalid *ocrValidationError
|
||||
if errors.As(vErr, &invalid) {
|
||||
r.out["ocr_rejection_reason"] = invalid.reason
|
||||
}
|
||||
resolve()
|
||||
return nil
|
||||
}
|
||||
if ocrText != "" {
|
||||
r.imageInfo.OCRText = ocrText
|
||||
r.out["ocr_status"] = "succeeded"
|
||||
r.out["ocr_chars"] = len([]rune(ocrText))
|
||||
r.out["ocr_preview"] = previewText(ocrText, 200)
|
||||
resolve()
|
||||
return nil
|
||||
}
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR returned no text for %s", r.payload.ImageURL)
|
||||
r.out["ocr_status"] = "no_text"
|
||||
r.out["ocr_skipped"] = "no_text"
|
||||
resolve()
|
||||
return nil
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/imageprep"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
|
|
@ -158,8 +159,8 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) (
|
|||
}
|
||||
|
||||
logger.Infof(ctx,
|
||||
"[ImageMultimodal] Processing image: chunk=%s, url=%s, ocr=%v, caption=%v, attrs=%v",
|
||||
payload.ChunkID, payload.ImageURL, payload.EnableOCR, payload.EnableCaption, payload.ImageAttrsEnabled)
|
||||
"[ImageMultimodal] Processing image: chunk=%s, url=%s, attrs=%v",
|
||||
payload.ChunkID, payload.ImageURL, payload.ImageAttrsEnabled)
|
||||
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
|
|
@ -212,7 +213,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) (
|
|||
// double-count and prematurely trigger post-process.
|
||||
var handleErr error
|
||||
defer func() {
|
||||
if handleErr == nil || isFinalAsynqAttempt(ctx) {
|
||||
// Finalize on success, on the final asynq retry, or when the error is
|
||||
// a permanent failure wrapped with asynq.SkipRetry. The SkipRetry
|
||||
// branch is essential: a SkipRetry error makes the executor break
|
||||
// before the retry counter reaches maxRetry, so isFinalAsynqAttempt
|
||||
// would otherwise return false and the parent knowledge would be
|
||||
// stranded in "processing" forever.
|
||||
if handleErr == nil || isFinalAsynqAttempt(ctx) || errors.Is(handleErr, asynq.SkipRetry) {
|
||||
if err := s.checkAndFinalizeAllImages(ctx, payload); err != nil && retErr == nil {
|
||||
retErr = err
|
||||
}
|
||||
|
|
@ -222,7 +229,6 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) (
|
|||
payload.ImageURL)
|
||||
}
|
||||
}()
|
||||
|
||||
vlmModel, vlmCfg, err := s.resolveVLM(ctx, payload.KnowledgeBaseID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
handleErr = fmt.Errorf("resolve VLM: %w", err)
|
||||
|
|
@ -238,17 +244,16 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) (
|
|||
imgOut["vlm_model_id"] = "legacy_inline"
|
||||
}
|
||||
|
||||
handleErr = s.processImage(ctx, &payload, vlmModel, vlmCfg, tracker, imgOut)
|
||||
handleErr = wrapSkipRetry(s.processImage(ctx, &payload, vlmModel, vlmCfg, tracker, imgOut))
|
||||
return handleErr
|
||||
}
|
||||
|
||||
// processImage runs the unified image action loop for one image. The selector
|
||||
// plans rounds from the payload switches; the dispatcher runs each round
|
||||
// (observation and caption share one VLM call); the result processor reads the
|
||||
// observation and, when the OCR policy wants it, appends an OCR round. The
|
||||
// observation must be known before any OCR call is spent — that ordering is
|
||||
// what makes "observe first, then act" possible: an image whose text is
|
||||
// reliably absent does not pay for OCR.
|
||||
// processImage runs one image pipeline for one image. The pipeline is chosen
|
||||
// from the payload switches and owns the whole control flow; this function only
|
||||
// hands it the run context and closes the span afterwards. The observation must
|
||||
// be known before any OCR call is spent — that ordering is what makes "observe
|
||||
// first, then act" possible: an image whose text is reliably absent does not pay
|
||||
// for OCR.
|
||||
//
|
||||
// An observation (describe) failure is not fatal. The attributes then stay at
|
||||
// their conservative defaults, which keeps OCR running, so a model that cannot
|
||||
|
|
@ -265,6 +270,10 @@ func (s *ImageMultimodalService) processImage(
|
|||
tracker SpanTracker,
|
||||
out types.JSONMap,
|
||||
) error {
|
||||
// Selected from the payload because a task in flight was enqueued for one
|
||||
// pipeline and must not be re-routed by an edit made since; the id also
|
||||
// lands on the trace, which reading image_info alone cannot do.
|
||||
pipeline := selectImagePipeline(payload)
|
||||
// Open a per-image subspan under the parent attempt's multimodal stage.
|
||||
// If the parent stage row is missing (legacy in-flight task, or the
|
||||
// upstream code shipped without span tracking), the tracker is a no-op so
|
||||
|
|
@ -281,13 +290,10 @@ func (s *ImageMultimodalService) processImage(
|
|||
"image_url": payload.ImageURL,
|
||||
"image_source_type": payload.ImageSourceType,
|
||||
"parent_chunk_id": payload.ChunkID,
|
||||
// Which pipeline the trace is looking at: observation_driven
|
||||
// (a describe round that also observes the attributes, then OCR
|
||||
// only if the policy says so) or caption_ocr (caption then OCR,
|
||||
// no observation). Reading image_info alone cannot tell a
|
||||
// caption_ocr run from an observation_driven run that skipped
|
||||
// OCR.
|
||||
"pipeline": string(types.PipelineModeFor(payload.ImageAttrsEnabled)),
|
||||
// Which pipeline ran. Reading image_info alone cannot tell a
|
||||
// default run from a smartocr run that observed and then
|
||||
// skipped OCR.
|
||||
"pipeline": string(pipeline.ID()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -303,9 +309,21 @@ func (s *ImageMultimodalService) processImage(
|
|||
}
|
||||
if handleErr == nil {
|
||||
tracker.EndSpan(ctx, imgSpan, out)
|
||||
} else if isFinalAsynqAttempt(ctx) {
|
||||
} else {
|
||||
// Any genuinely failed attempt must close its span red, not
|
||||
// leave it dangling as "in progress". The span is a Generation
|
||||
// subspan, so FailSpan here cannot poison the parent stage or
|
||||
// the attempt status (cascade only fires for Stage-kind spans);
|
||||
// it merely records this attempt's outcome and lets a later
|
||||
// retry open its own span under the new attempt. A caller
|
||||
// cancellation gets its own code: the run did not fail, it was
|
||||
// walked away from.
|
||||
code := "MULTIMODAL_VLM_FAILED"
|
||||
if errors.Is(handleErr, context.Canceled) {
|
||||
code = "MULTIMODAL_CANCELLED"
|
||||
}
|
||||
tracker.FailSpan(ctx, imgSpan,
|
||||
"MULTIMODAL_VLM_FAILED",
|
||||
code,
|
||||
handleErr.Error(),
|
||||
handleErr)
|
||||
}
|
||||
|
|
@ -331,51 +349,32 @@ func (s *ImageMultimodalService) processImage(
|
|||
OriginalURL: payload.ImageURL,
|
||||
}
|
||||
|
||||
// --- Unified action loop: the decision + dispatch half of the image
|
||||
// pipeline framework. Every image step is a normal action (caption,
|
||||
// observation, ocr). The selector plans rounds from the payload switches;
|
||||
// the dispatcher runs each round (observation + caption share one VLM call);
|
||||
// the result processor reads the observation and, when OCR is wanted,
|
||||
// appends an ocr round - the "observe, decide, then act" closure. The VLM
|
||||
// transport here is a direct-call shim; PR-E routes it through a request
|
||||
// manager without touching this dispatch logic.
|
||||
if !payload.ImageAttrsEnabled && !payload.EnableOCR {
|
||||
// Caption+OCR mode with OCR off: recorded so the trace explains the
|
||||
// missing OCR chunk rather than leaving it to be inferred from an
|
||||
// absence.
|
||||
out["ocr_skipped"] = "disabled"
|
||||
// The pipeline runs to completion, including the decision it makes about
|
||||
// OCR. Nothing below re-reads its result except the chunk building that
|
||||
// follows. The tracker and the image span travel with the run context so
|
||||
// the OCR action can raise and resolve its own subspan exactly like the
|
||||
// action-loop layout did (a failed OCR fails the .ocr subspan and the
|
||||
// outcome summary, without failing or retrying the whole image).
|
||||
rc := &runContext{
|
||||
payload: payload,
|
||||
pipelineID: string(pipeline.ID()),
|
||||
params: payload.ImagePipelineParams,
|
||||
declared: pipeline.Fields(),
|
||||
model: vlmModel,
|
||||
imageBytes: imgBytes,
|
||||
vlmCfg: vlmCfg,
|
||||
imageInfo: &imageInfo,
|
||||
out: out,
|
||||
tracker: tracker,
|
||||
imgSpan: imgSpan,
|
||||
}
|
||||
plan := selectImageActionRounds(payload)
|
||||
for i := 0; i < len(plan); i++ {
|
||||
round := plan[i]
|
||||
var ocrSpan *Span
|
||||
if round.Contains(types.ActionOCR) && imgSpan != nil {
|
||||
ocrSpan = tracker.BeginSubSpan(ctx, imgSpan, imgSpan.Name+".ocr", types.SpanKindGeneration, nil)
|
||||
}
|
||||
s.executeActionRound(ctx, payload, vlmModel, imgBytes, vlmCfg, &imageInfo, out, round)
|
||||
if ocrSpan != nil {
|
||||
if message, failed := out["ocr_error"].(string); failed {
|
||||
code, _ := out["ocr_error_code"].(string)
|
||||
tracker.FailSpan(ctx, ocrSpan, code, message, nil)
|
||||
} else {
|
||||
tracker.EndSpan(ctx, ocrSpan, types.JSONMap{
|
||||
"status": out["ocr_status"], "chars": out["ocr_chars"],
|
||||
})
|
||||
}
|
||||
}
|
||||
// Result processor: only an observation round feeds the OCR decision.
|
||||
if round.Contains(types.ActionObservation) {
|
||||
ocrWanted := payload.EnableOCR && DecideOCR(imageInfo.Attrs, payload.ImageActions)
|
||||
out["attr_policy"] = types.JSONMap{"ocr": ocrWanted}
|
||||
out["image_attrs"] = imageInfo.Attrs.Attrs
|
||||
if ocrWanted {
|
||||
plan = append(plan, ActionRound{Actions: []types.ActionKind{types.ActionOCR}})
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Skipping OCR for %s (attrs=%v)",
|
||||
payload.ImageURL, imageInfo.Attrs.Attrs)
|
||||
out["ocr_skipped"] = "attr_policy"
|
||||
}
|
||||
}
|
||||
if err := pipeline.Run(ctx, rc); err != nil {
|
||||
// Surface the pipeline failure on this image's span instead of
|
||||
// letting the deferred finalize misread it as success. Without
|
||||
// this assignment the span below keeps handleErr == nil and the
|
||||
// UI shows a green (done) result for a genuinely failed image.
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
|
||||
// Build child chunks for OCR and caption results
|
||||
|
|
@ -400,7 +399,9 @@ func (s *ImageMultimodalService) processImage(
|
|||
})
|
||||
}
|
||||
|
||||
if payload.EnableCaption && imageInfo.Caption != "" {
|
||||
// A caption that reached the slot came from an action this run executed, so
|
||||
// an empty slot needs no second question asked of it.
|
||||
if imageInfo.Caption != "" {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
|
|
@ -418,19 +419,50 @@ func (s *ImageMultimodalService) processImage(
|
|||
})
|
||||
}
|
||||
out["chunks_created"] = len(newChunks)
|
||||
if out["ocr_status"] == "failed" {
|
||||
out["outcome"] = "partial_failure"
|
||||
if len(newChunks) == 0 {
|
||||
out["outcome"] = "failed"
|
||||
|
||||
// Nothing was produced and a requested extraction failed: the image
|
||||
// has no image_info, so it will not appear in the gallery, and a silent
|
||||
// "skipped" success would hide a dead endpoint behind a green trace.
|
||||
// Report the failure to the task layer — the caller wraps permanent
|
||||
// failures (dead endpoint, bad key) with SkipRetry, transient ones burn
|
||||
// at most the asynq retry budget, and the deferred finalize marks this
|
||||
// image's span red (MULTIMODAL_CANCELLED when the run was walked away
|
||||
// from). The #4132 refinement: failures stay recorded-not-propagated only
|
||||
// while there is content to protect — a valid caption must not be
|
||||
// duplicated by an OCR retry; with nothing produced there is nothing to
|
||||
// protect, so the image fails honestly.
|
||||
extractionFailed := out["caption_error"] != nil || out["ocr_status"] == "failed" ||
|
||||
out["ocr_skipped"] == "no_ocr_after_observation_failed"
|
||||
if len(newChunks) == 0 && extractionFailed {
|
||||
out["outcome"] = "failed"
|
||||
if ctx.Err() != nil {
|
||||
handleErr = fmt.Errorf("image processing cancelled with no content produced: %w", ctx.Err())
|
||||
return handleErr
|
||||
}
|
||||
if rc.vlmErr != nil {
|
||||
// The typed VLM error travels with the verdict so the task
|
||||
// layer can tell a permanent failure (SkipRetry) from a
|
||||
// transient one.
|
||||
handleErr = fmt.Errorf("image produced no content: %w", rc.vlmErr)
|
||||
} else {
|
||||
handleErr = fmt.Errorf("image produced no content (ocr_status=%v, caption_error=%v)",
|
||||
out["ocr_status"], out["caption_error"])
|
||||
}
|
||||
return handleErr
|
||||
}
|
||||
|
||||
if out["ocr_status"] == "failed" {
|
||||
// Some content exists (a valid caption was persisted) — the failure
|
||||
// is recorded on the .ocr subspan and the outcome, and the image
|
||||
// stays a success at the task level (#4132).
|
||||
out["outcome"] = "partial_failure"
|
||||
}
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
// Deferred finalize will count this image on success.
|
||||
// Deferred finalize will count this image on success. A genuine
|
||||
// blank (OCR answered no_text) or a policy-declined image keeps its
|
||||
// trace-level skip.
|
||||
out["skipped"] = "no_extracted_content"
|
||||
if out["ocr_status"] == "failed" {
|
||||
out["skipped"] = "ocr_failed"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -509,82 +541,6 @@ func applyImageObservation(
|
|||
out["caption_preview"] = previewText(description, 200)
|
||||
}
|
||||
|
||||
// runImageOCR runs the second pipeline round: text extraction at full
|
||||
// resolution. Both pipelines share it — the observation-driven one reaches it
|
||||
// only when the attribute policy allows it, caption+OCR mode runs it for every
|
||||
// image.
|
||||
func (s *ImageMultimodalService) runImageOCR(
|
||||
ctx context.Context,
|
||||
payload *types.ImageMultimodalPayload,
|
||||
vlmModel vlm.VLM,
|
||||
imgBytes []byte,
|
||||
imageInfo *types.ImageInfo,
|
||||
out types.JSONMap,
|
||||
vlmCfg types.VLMConfig,
|
||||
) {
|
||||
ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
|
||||
Name: "image.ocr", Metadata: map[string]interface{}{"model_id": vlmModel.GetModelID()},
|
||||
})
|
||||
var failure error
|
||||
defer func() {
|
||||
span.Finish(map[string]interface{}{
|
||||
"status": out["ocr_status"], "error_code": out["ocr_error_code"],
|
||||
"rejection_reason": out["ocr_rejection_reason"], "chars": out["ocr_chars"],
|
||||
"raw_chars": out["ocr_raw_chars"],
|
||||
}, nil, failure)
|
||||
}()
|
||||
// A failed extraction must never leave text from a previous attempt behind.
|
||||
imageInfo.OCRText = ""
|
||||
out["ocr_chars"] = 0
|
||||
// The OCR prompt is system-owned: knowledge base custom instructions must
|
||||
// never reach it, or free-form business rules compete with the "No text
|
||||
// content" contract and poison image_ocr chunks. buildVLMOCRPrompt picks
|
||||
// the scanned-PDF or default prompt and ignores vlmCfg on purpose.
|
||||
prompt := buildVLMOCRPrompt(payload.ImageSourceType, vlmCfg)
|
||||
if payload.ImageSourceType == "scanned_pdf" {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL)
|
||||
out["ocr_prompt"] = "scanned_pdf"
|
||||
} else {
|
||||
out["ocr_prompt"] = "default"
|
||||
}
|
||||
|
||||
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
|
||||
if ocrErr != nil {
|
||||
failure = ocrErr
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
|
||||
out["ocr_status"] = "failed"
|
||||
out["ocr_error_code"] = "OCR_REQUEST_FAILED"
|
||||
if errors.Is(ocrErr, vlm.ErrTruncatedCompletion) {
|
||||
out["ocr_error_code"] = "OCR_TRUNCATED"
|
||||
}
|
||||
out["ocr_error"] = ocrErr.Error()
|
||||
return
|
||||
}
|
||||
out["ocr_raw_chars"] = len([]rune(ocrText))
|
||||
ocrText, ocrErr = validateOCRText(ocrText)
|
||||
if ocrErr != nil {
|
||||
failure = ocrErr
|
||||
out["ocr_status"] = "failed"
|
||||
out["ocr_error_code"] = "OCR_INVALID_OUTPUT"
|
||||
out["ocr_error"] = ocrErr.Error()
|
||||
var invalid *ocrValidationError
|
||||
if errors.As(ocrErr, &invalid) {
|
||||
out["ocr_rejection_reason"] = invalid.reason
|
||||
}
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR rejected for %s: %v", payload.ImageURL, ocrErr)
|
||||
return
|
||||
}
|
||||
if ocrText != "" {
|
||||
imageInfo.OCRText = ocrText
|
||||
out["ocr_status"] = "succeeded"
|
||||
out["ocr_chars"] = len([]rune(ocrText))
|
||||
out["ocr_preview"] = previewText(ocrText, 200)
|
||||
} else {
|
||||
out["ocr_status"] = "no_text"
|
||||
out["ocr_skipped"] = "no_text"
|
||||
}
|
||||
}
|
||||
|
||||
// shouldDropOrphanedMultimodal reports whether the task should exit without
|
||||
// retrying. True for user-cancelled/deleting knowledge, or when the parent
|
||||
// knowledge / knowledge-base row no longer exists (deleted while queue entries
|
||||
|
|
@ -630,6 +586,63 @@ func (s *ImageMultimodalService) shouldDropOrphanedMultimodal(
|
|||
// Returns false when the values are unavailable (e.g. when the handler is
|
||||
// invoked outside an asynq worker, as in unit tests). Treating that case as
|
||||
// "not final" keeps test ergonomics — tests should drive finalize explicitly.
|
||||
// isPermanentVLMFailure reports whether an error returned by the image run is
|
||||
// a permanent failure that must NOT be retried by the task layer. Retrying
|
||||
// such an error only burns queue capacity and re-hits the same dead end, so
|
||||
// the caller wraps it with asynq.SkipRetry to archive the task immediately.
|
||||
//
|
||||
// Permanent (SkipRetry):
|
||||
// - a 4xx client error (bad key, oversized image, unsupported request): the
|
||||
// request cannot succeed on retry;
|
||||
// - a connection-level failure (connection refused / DNS / TLS / first-byte
|
||||
// timeout) surfaced as an api.TransportError in the "send request" phase:
|
||||
// the endpoint is unreachable, so every retry fails identically.
|
||||
//
|
||||
// Retryable (leave to the task-layer backoff):
|
||||
// - 5xx, 408, 429;
|
||||
// - a mid-stream transport error ("read response"): the stream broke after
|
||||
// the server answered, which is transient;
|
||||
// - anything else (unknown / context errors): safest to let the task retry.
|
||||
//
|
||||
// This is deliberately decoupled from the vlm package: it inspects only the
|
||||
// stable api.* error types the manager always propagates. The manager wraps
|
||||
// the original api error with %w, so errors.As still reaches it here.
|
||||
func isPermanentVLMFailure(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var httpErr *api.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
switch {
|
||||
case httpErr.StatusCode == 429, // Too Many Requests
|
||||
httpErr.StatusCode == 408: // Request Timeout
|
||||
return false
|
||||
case httpErr.StatusCode >= 500:
|
||||
return false
|
||||
case httpErr.StatusCode >= 400:
|
||||
return true // permanent client error
|
||||
}
|
||||
}
|
||||
var transportErr *api.TransportError
|
||||
if errors.As(err, &transportErr) {
|
||||
// "send request" = failure before any response (connect / TLS /
|
||||
// first-byte timeout): endpoint is down. "read response" = the stream
|
||||
// broke mid-flight, which is transient.
|
||||
return transportErr.Op == "send request"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// wrapSkipRetry marks a permanent VLM failure so the task layer archives the
|
||||
// task immediately instead of retrying it. Retryable errors are returned
|
||||
// unchanged.
|
||||
func wrapSkipRetry(err error) error {
|
||||
if err == nil || !isPermanentVLMFailure(err) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", asynq.SkipRetry, err)
|
||||
}
|
||||
|
||||
func isFinalAsynqAttempt(ctx context.Context) bool {
|
||||
retried, ok := asynq.GetRetryCount(ctx)
|
||||
if ok {
|
||||
|
|
|
|||
|
|
@ -40,6 +40,14 @@ func (f *attrsFakeVLM) Predict(_ context.Context, imgBytes [][]byte, prompt stri
|
|||
return f.reply(prompt, len(imgBytes))
|
||||
}
|
||||
|
||||
// PredictWithOptions is the per-call form; a fake answers it exactly as Predict
|
||||
// does, since the tests pin the prompt and the call count, not the switch.
|
||||
func (f *attrsFakeVLM) PredictWithOptions(
|
||||
_ context.Context, imgBytes [][]byte, prompt string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return f.Predict(context.Background(), imgBytes, prompt)
|
||||
}
|
||||
|
||||
func (f *attrsFakeVLM) GetModelName() string { return "attrs-fake-vlm" }
|
||||
func (f *attrsFakeVLM) GetModelID() string { return "attrs-fake-vlm-id" }
|
||||
|
||||
|
|
@ -300,13 +308,15 @@ func TestProcessImageObservesAndDescribes(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -373,13 +383,15 @@ func TestProcessImageMarksObservationFailed(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -421,13 +433,16 @@ func TestProcessImageSkipsOCRForQuietAttr(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true, // whole-task switch stays on; the observation is what vetoes
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
// The ceiling stays on; the observation is what vetoes.
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -474,13 +489,15 @@ func TestProcessImageKeepsAttrsButDropsCaptionWhenCaptionsAreOff(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: false,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: false,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -533,13 +550,15 @@ func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -597,13 +616,15 @@ func TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
@ -645,13 +666,15 @@ func TestProcessImageCaptionOCRPipeline(t *testing.T) {
|
|||
}
|
||||
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
captionFieldKeyEnableCaption: true,
|
||||
captionFieldKeyEnableOCR: true,
|
||||
},
|
||||
ImageAttrsEnabled: false,
|
||||
// An action table is present but must be ignored: caption+OCR mode has
|
||||
// no observation to feed it.
|
||||
|
|
@ -695,13 +718,15 @@ func TestProcessImageReturnsUnreadableImageError(t *testing.T) {
|
|||
|
||||
fake := &attrsFakeVLM{}
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
TenantID: 1,
|
||||
KnowledgeID: "k-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
ImageURL: "local://img/0.png",
|
||||
ChunkID: "chunk-a",
|
||||
ImagePipelineParams: map[string]any{
|
||||
smartFieldKeyAllowOCR: true,
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
},
|
||||
ImageAttrsEnabled: true,
|
||||
ImageActions: types.DefaultImageActions(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,211 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsPermanentVLMFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"generic", errors.New("boom"), false},
|
||||
{"context deadline", context.DeadlineExceeded, false},
|
||||
{"context cancelled", context.Canceled, false},
|
||||
{"401 permanent", &api.HTTPError{StatusCode: 401}, true},
|
||||
{"403 permanent", &api.HTTPError{StatusCode: 403}, true},
|
||||
{"404 permanent", &api.HTTPError{StatusCode: 404}, true},
|
||||
{"400 permanent", &api.HTTPError{StatusCode: 400}, true},
|
||||
{"413 permanent", &api.HTTPError{StatusCode: 413}, true},
|
||||
{"429 retryable", &api.HTTPError{StatusCode: 429}, false},
|
||||
{"408 retryable", &api.HTTPError{StatusCode: 408}, false},
|
||||
{"500 retryable", &api.HTTPError{StatusCode: 500}, false},
|
||||
{"503 retryable", &api.HTTPError{StatusCode: 503}, false},
|
||||
{
|
||||
"transport send-request down",
|
||||
&api.TransportError{Op: "send request", Err: errors.New("connection refused")}, true,
|
||||
},
|
||||
{
|
||||
"transport read-response transient",
|
||||
&api.TransportError{Op: "read response", Err: errors.New("unexpected EOF")}, false,
|
||||
},
|
||||
// The manager wraps the original api error with %w, so the underlying
|
||||
// api error must still be detectable.
|
||||
{
|
||||
"wrapped send-request",
|
||||
fmt.Errorf("endpoint down: %w", &api.TransportError{Op: "send request", Err: errors.New("i/o timeout")}),
|
||||
true,
|
||||
},
|
||||
{"wrapped 401", fmt.Errorf("verdict: %w", &api.HTTPError{StatusCode: 401}), true},
|
||||
{"wrapped 500", fmt.Errorf("verdict: %w", &api.HTTPError{StatusCode: 500}), false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isPermanentVLMFailure(tc.err); got != tc.want {
|
||||
t.Errorf("%s: isPermanentVLMFailure = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapSkipRetry(t *testing.T) {
|
||||
// Permanent failures get the asynq.SkipRetry sentinel and keep the original
|
||||
// error inspectable.
|
||||
down := wrapSkipRetry(&api.TransportError{Op: "send request", Err: errors.New("connection refused")})
|
||||
if !errors.Is(down, asynq.SkipRetry) {
|
||||
t.Errorf("down error = %v, want it to carry asynq.SkipRetry", down)
|
||||
}
|
||||
var te *api.TransportError
|
||||
if !errors.As(down, &te) {
|
||||
t.Errorf("down error = %v, want the original TransportError still inspectable", down)
|
||||
}
|
||||
|
||||
perm := wrapSkipRetry(&api.HTTPError{StatusCode: 401})
|
||||
if !errors.Is(perm, asynq.SkipRetry) {
|
||||
t.Errorf("perm error = %v, want asynq.SkipRetry", perm)
|
||||
}
|
||||
|
||||
// Retryable failures are returned unchanged (errors.Is must be false).
|
||||
retry := wrapSkipRetry(&api.HTTPError{StatusCode: 500})
|
||||
if errors.Is(retry, asynq.SkipRetry) {
|
||||
t.Errorf("retry(500) error = %v, want NO asynq.SkipRetry", retry)
|
||||
}
|
||||
retry2 := wrapSkipRetry(&api.TransportError{Op: "read response", Err: errors.New("EOF")})
|
||||
if errors.Is(retry2, asynq.SkipRetry) {
|
||||
t.Errorf("retry(read-response) error = %v, want NO asynq.SkipRetry", retry2)
|
||||
}
|
||||
|
||||
if got := wrapSkipRetry(nil); got != nil {
|
||||
t.Errorf("wrapSkipRetry(nil) = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllFailedImageIsAnError pins the #4132 refinement: a run that produced
|
||||
// NOTHING (no caption, no OCR) and had every VLM call fail is a real failure —
|
||||
// the image would be invisible in the gallery, so a silent "skipped" success
|
||||
// would hide a dead endpoint behind a green trace. A run that DID produce a
|
||||
// caption keeps the recorded-not-propagated semantics.
|
||||
func TestAllFailedImageIsAnError(t *testing.T) {
|
||||
newDown := func() *attrsFakeVLM {
|
||||
return &attrsFakeVLM{reply: func(string, int) (string, error) {
|
||||
return "", &api.TransportError{Op: "send request", Err: errors.New("connection refused")}
|
||||
}}
|
||||
}
|
||||
|
||||
t.Run("caption off + ocr failed = error", func(t *testing.T) {
|
||||
repo := &attrsChunkRepo{}
|
||||
svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
|
||||
tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
|
||||
out := types.JSONMap{}
|
||||
err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png",
|
||||
ImagePipelineParams: map[string]any{"enable_caption": false, "enable_ocr": true},
|
||||
Attempt: 1,
|
||||
}, newDown(), types.VLMConfig{}, tracker, out)
|
||||
require.Error(t, err, "nothing was produced and every call failed: the image must not pass as skipped")
|
||||
// The typed VLM error must survive the wrap so Handle's wrapSkipRetry
|
||||
// can classify the run (covered by TestWrapSkipRetry — this test
|
||||
// calls processImage directly, before that wrap).
|
||||
var te *api.TransportError
|
||||
require.ErrorAs(t, err, &te)
|
||||
require.Equal(t, "failed", out["outcome"])
|
||||
require.Equal(t, "MULTIMODAL_VLM_FAILED", tracker.failed["multimodal.image[0]"],
|
||||
"the image span must close red")
|
||||
})
|
||||
|
||||
t.Run("caption on + ocr failed = recorded, not propagated", func(t *testing.T) {
|
||||
repo := &attrsChunkRepo{}
|
||||
svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
|
||||
model := &attrsFakeVLM{reply: func(prompt string, _ int) (string, error) {
|
||||
if prompt == vlmOCRPrompt {
|
||||
return "", &api.TransportError{Op: "send request", Err: errors.New("connection refused")}
|
||||
}
|
||||
return "A game screenshot.", nil
|
||||
}}
|
||||
tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
|
||||
out := types.JSONMap{}
|
||||
err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png",
|
||||
ImagePipelineParams: map[string]any{"enable_caption": true, "enable_ocr": true},
|
||||
Attempt: 1,
|
||||
}, model, types.VLMConfig{}, tracker, out)
|
||||
require.NoError(t, err, "a valid caption must not be duplicated by an OCR retry")
|
||||
require.Equal(t, "partial_failure", out["outcome"])
|
||||
})
|
||||
}
|
||||
|
||||
// TestCancelledAllFailedImageIsCancelled pins the cancellation distinction: a
|
||||
// run walked away from with nothing produced closes its span
|
||||
// MULTIMODAL_CANCELLED, and the error does NOT carry asynq.SkipRetry (the
|
||||
// executor is revoking the task anyway; the classification must not pretend
|
||||
// the endpoint is permanent).
|
||||
func TestCancelledAllFailedImageIsCancelled(t *testing.T) {
|
||||
repo := &attrsChunkRepo{}
|
||||
svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
|
||||
model := &attrsFakeVLM{reply: func(string, int) (string, error) {
|
||||
return "", errors.New("model unavailable")
|
||||
}}
|
||||
tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
|
||||
out := types.JSONMap{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := svc.processImage(ctx, &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png",
|
||||
ImagePipelineParams: map[string]any{"enable_caption": false, "enable_ocr": true},
|
||||
Attempt: 1,
|
||||
}, model, types.VLMConfig{}, tracker, out)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.NotErrorIs(t, err, asynq.SkipRetry)
|
||||
require.Equal(t, "MULTIMODAL_CANCELLED", tracker.failed["multimodal.image[0]"])
|
||||
}
|
||||
|
||||
func TestCaptionOnlyFailureFailsImage(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
permanent bool
|
||||
}{
|
||||
{"retryable", &api.HTTPError{StatusCode: 503}, false},
|
||||
{"permanent", &api.HTTPError{StatusCode: 401}, true},
|
||||
{"cancelled", context.Canceled, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &attrsChunkRepo{}
|
||||
svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
|
||||
model := &attrsFakeVLM{reply: func(string, int) (string, error) { return "", tc.err }}
|
||||
tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
|
||||
out := types.JSONMap{}
|
||||
ctx := context.Background()
|
||||
if errors.Is(tc.err, context.Canceled) {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
cancel()
|
||||
}
|
||||
err := svc.processImage(ctx, &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png", Attempt: 1,
|
||||
ImagePipelineID: types.ImagePipelineDefault,
|
||||
ImagePipelineParams: map[string]any{"enable_caption": true, "enable_ocr": false},
|
||||
}, model, types.VLMConfig{}, tracker, out)
|
||||
require.ErrorIs(t, err, tc.err)
|
||||
require.Equal(t, "failed", out["outcome"])
|
||||
require.Equal(t, 0, out["chunks_created"])
|
||||
require.NotContains(t, out, "skipped")
|
||||
require.NotContains(t, out, "ocr_status")
|
||||
require.Equal(t, tc.permanent, errors.Is(wrapSkipRetry(err), asynq.SkipRetry))
|
||||
code := "MULTIMODAL_VLM_FAILED"
|
||||
if errors.Is(tc.err, context.Canceled) {
|
||||
code = "MULTIMODAL_CANCELLED"
|
||||
}
|
||||
require.Equal(t, code, tracker.failed["multimodal.image[0]"])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// errorVLM fails every prediction, standing in for a model endpoint that
|
||||
// answered non-2xx: a 429 while the provider throttles, a 5xx, a timeout, or a
|
||||
// 4xx such as 400/413/431 the endpoint rejected the request with.
|
||||
type errorVLM struct{ err error }
|
||||
|
||||
func (m *errorVLM) Predict(_ context.Context, _ [][]byte, _ string) (string, error) {
|
||||
return "", m.err
|
||||
}
|
||||
|
||||
func (m *errorVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, _ string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return "", m.err
|
||||
}
|
||||
|
||||
func (m *errorVLM) GetModelName() string { return "error" }
|
||||
func (m *errorVLM) GetModelID() string { return "error" }
|
||||
|
||||
var _ vlm.VLM = (*errorVLM)(nil)
|
||||
|
||||
// emptyVLM answers every prediction with no text: the call succeeded, but the
|
||||
// output carries nothing readable.
|
||||
type emptyVLM struct{}
|
||||
|
||||
func (emptyVLM) Predict(_ context.Context, _ [][]byte, _ string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (emptyVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, _ string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (emptyVLM) GetModelName() string { return "empty" }
|
||||
func (emptyVLM) GetModelID() string { return "empty" }
|
||||
|
||||
var _ vlm.VLM = emptyVLM{}
|
||||
|
||||
func ocrRunContext(model vlm.VLM) *runContext {
|
||||
return &runContext{
|
||||
payload: &types.ImageMultimodalPayload{ImageSourceType: "image"},
|
||||
model: model,
|
||||
imageBytes: []byte("fake image bytes"),
|
||||
vlmCfg: types.VLMConfig{},
|
||||
imageInfo: &types.ImageInfo{},
|
||||
out: types.JSONMap{},
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunOCRActionRecordsFailureWithoutFailing pins the #4132 contract: a
|
||||
// failed OCR call is RECORDED (status, stable error code, raw message) but
|
||||
// must NOT fail the action — retrying the whole image would re-run a caption
|
||||
// that already succeeded and duplicate its chunk. The failure stays visible
|
||||
// through the recorded fields and the outcome summary, never through a
|
||||
// swallowed empty result.
|
||||
func TestRunOCRActionRecordsFailureWithoutFailing(t *testing.T) {
|
||||
r := ocrRunContext(&errorVLM{err: errors.New("429 too many requests")})
|
||||
|
||||
if err := runOCRAction(context.Background(), r); err != nil {
|
||||
t.Fatalf("runOCRAction error = %v, want nil (failure is recorded, not propagated)", err)
|
||||
}
|
||||
if got := r.out["ocr_status"]; got != "failed" {
|
||||
t.Errorf("ocr_status = %v, want failed", got)
|
||||
}
|
||||
if got := r.out["ocr_error_code"]; got != "OCR_REQUEST_FAILED" {
|
||||
t.Errorf("ocr_error_code = %v, want OCR_REQUEST_FAILED", got)
|
||||
}
|
||||
if _, ok := r.out["ocr_error"]; !ok {
|
||||
t.Errorf("ocr_error not recorded for a failed call")
|
||||
}
|
||||
if _, skipped := r.out["ocr_skipped"]; skipped {
|
||||
t.Errorf(`a failed call must not read as skipped: %v`, r.out["ocr_skipped"])
|
||||
}
|
||||
if got := r.imageInfo.OCRText; got != "" {
|
||||
t.Errorf("a failed call must leave no OCR text behind, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunOCRActionCodesTruncation pins the budget-exhaustion branch: an error
|
||||
// wrapping vlm.ErrTruncatedCompletion is re-coded OCR_TRUNCATED so the trace
|
||||
// distinguishes "the model ran out of budget" from "the request failed".
|
||||
func TestRunOCRActionCodesTruncation(t *testing.T) {
|
||||
r := ocrRunContext(&errorVLM{err: fmt.Errorf("request: %w", vlm.ErrTruncatedCompletion)})
|
||||
|
||||
if err := runOCRAction(context.Background(), r); err != nil {
|
||||
t.Fatalf("runOCRAction error = %v, want nil", err)
|
||||
}
|
||||
if got := r.out["ocr_error_code"]; got != "OCR_TRUNCATED" {
|
||||
t.Errorf("ocr_error_code = %v, want OCR_TRUNCATED", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunOCRActionRejectsEmptyOutput pins the validation half: a successful
|
||||
// call whose output has nothing readable (empty, or a known-empty reply the
|
||||
// sanitizer does not whitelist) is an INVALID output — failed with
|
||||
// OCR_INVALID_OUTPUT — and not a success. A model that explicitly answers "no
|
||||
// text" is the no_text branch the validation suite covers.
|
||||
func TestRunOCRActionRejectsEmptyOutput(t *testing.T) {
|
||||
r := ocrRunContext(emptyVLM{})
|
||||
|
||||
if err := runOCRAction(context.Background(), r); err != nil {
|
||||
t.Fatalf("runOCRAction returned %v for an empty-but-successful answer", err)
|
||||
}
|
||||
if got := r.out["ocr_status"]; got != "failed" {
|
||||
t.Errorf("ocr_status = %v, want failed (empty output is invalid, not no_text)", got)
|
||||
}
|
||||
if got := r.out["ocr_error_code"]; got != "OCR_INVALID_OUTPUT" {
|
||||
t.Errorf("ocr_error_code = %v, want OCR_INVALID_OUTPUT", got)
|
||||
}
|
||||
if _, ok := r.out["ocr_error"]; !ok {
|
||||
t.Errorf("ocr_error not recorded for invalid output")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultPipelineRecordsOCRFailure is the #4064 regression at the level it
|
||||
// was reported: the manual pipeline used to swallow the OCR failure, so
|
||||
// processImage found neither caption nor text and finished the image as done.
|
||||
// The failure now surfaces as recorded fields (and a failed .ocr subspan in
|
||||
// the full processImage path) while pipeline.Run itself stays green — the
|
||||
// outcome summary, not a task retry, is what carries the signal.
|
||||
func TestDefaultPipelineRecordsOCRFailure(t *testing.T) {
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
ImageAttrsEnabled: false,
|
||||
ImagePipelineID: types.ImagePipelineDefault,
|
||||
}
|
||||
pipeline := selectImagePipeline(payload)
|
||||
if pipeline == nil {
|
||||
t.Fatal("default pipeline not registered")
|
||||
}
|
||||
r := &runContext{
|
||||
payload: payload,
|
||||
declared: pipeline.Fields(),
|
||||
model: &errorVLM{err: errors.New("503 service unavailable")},
|
||||
imageBytes: []byte("fake image bytes"),
|
||||
vlmCfg: types.VLMConfig{},
|
||||
imageInfo: &types.ImageInfo{},
|
||||
out: types.JSONMap{},
|
||||
}
|
||||
|
||||
if err := pipeline.Run(context.Background(), r); err != nil {
|
||||
t.Fatalf("pipeline.Run error = %v, want nil (failure is recorded, not propagated)", err)
|
||||
}
|
||||
if got := r.out["ocr_status"]; got != "failed" {
|
||||
t.Errorf("ocr_status = %v, want failed", got)
|
||||
}
|
||||
if got := r.out["ocr_error"]; got == "" {
|
||||
t.Error("ocr_error not recorded for a failed call")
|
||||
}
|
||||
}
|
||||
|
|
@ -45,7 +45,7 @@ func TestLiveImageOCRValidation(t *testing.T) {
|
|||
defer cancel()
|
||||
start := time.Now()
|
||||
err = svc.processImage(ctx, &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://fixture", EnableOCR: true, Attempt: 1,
|
||||
ImageURL: "local://fixture", Attempt: 1,
|
||||
}, model, types.VLMConfig{}, tracker, out)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, tracker.failed)
|
||||
|
|
|
|||
|
|
@ -69,10 +69,30 @@ func TestImageOCRRejectsInvalidContentBeforePersistence(t *testing.T) {
|
|||
}}
|
||||
tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
|
||||
out := types.JSONMap{}
|
||||
// The pipeline framework carries the caption/OCR switches as
|
||||
// the manual pipeline's own tunables (both default true), the
|
||||
// whole-task EnableOCR/EnableCaption pair of the action loop
|
||||
// having been superseded by the image_pipeline field.
|
||||
err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png", EnableOCR: true, EnableCaption: caption, Attempt: 1,
|
||||
ImageURL: "local://test.png",
|
||||
ImagePipelineParams: map[string]any{"enable_caption": caption, "enable_ocr": true},
|
||||
Attempt: 1,
|
||||
}, model, types.VLMConfig{}, tracker, out)
|
||||
require.NoError(t, err, "bad OCR must not retry the whole image and duplicate a valid caption")
|
||||
if caption {
|
||||
// A valid caption exists: the OCR failure is recorded, not
|
||||
// propagated — failing the image would retry it and
|
||||
// duplicate the caption chunk (#4132).
|
||||
require.NoError(t, err)
|
||||
} else if tc.code != "" {
|
||||
// Caption off + OCR failed = every VLM call failed and
|
||||
// nothing was produced: the image fails honestly instead
|
||||
// of passing as a silent skip (there is no caption to
|
||||
// protect, so the #4132 no-duplicate rationale is void).
|
||||
require.Error(t, err, "bad OCR with no caption must fail the image")
|
||||
} else {
|
||||
// OCR answered no_text: a genuine blank, not a failure.
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, tc.status, out["ocr_status"])
|
||||
require.Equal(t, 0, out["ocr_chars"])
|
||||
require.NotContains(t, out, "ocr_preview")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func init() { registerImagePipeline(defaultPipeline{}) }
|
||||
|
||||
// defaultPipeline is the manual path: the switches on the panel decide which
|
||||
// parsing actions run, and each chosen action runs for every image, with no
|
||||
// observation and no policy. The id is "default" on purpose — it is what a
|
||||
// knowledge base falls back to when nothing else was picked, and it is where
|
||||
// every registered action becomes a manually selectable switch as they land.
|
||||
type defaultPipeline struct{}
|
||||
|
||||
func (defaultPipeline) ID() types.ImagePipelineID { return types.ImagePipelineDefault }
|
||||
|
||||
func (defaultPipeline) Name() string { return "Manual: pick the parsing actions" }
|
||||
|
||||
func (defaultPipeline) Description() string {
|
||||
return "The user switches the parsing actions on or off to fit the task."
|
||||
}
|
||||
|
||||
// Field keys of this pipeline. They are declared as constants because the
|
||||
// default lives in the same struct as the key: a panel that renders the field
|
||||
// under one spelling and the run that reads it under another is the exact bug
|
||||
// this shape is meant to make impossible.
|
||||
const (
|
||||
captionFieldKeyEnableCaption = "enable_caption"
|
||||
captionFieldKeyEnableOCR = "enable_ocr"
|
||||
)
|
||||
|
||||
// Fields declares the switches the panel renders for this pipeline. Caption and
|
||||
// OCR default to true, which is what a knowledge base configured before these
|
||||
// fields existed gets: both actions run, as before. The two thinking switches
|
||||
// default to false for the opposite reason — they are new work, and an action
|
||||
// that reasons costs a second full pass before it can say anything at all.
|
||||
func (defaultPipeline) Fields() []types.ImageFieldDef {
|
||||
return []types.ImageFieldDef{
|
||||
{
|
||||
Key: captionFieldKeyEnableCaption,
|
||||
Type: types.ImageFieldTypeBool,
|
||||
Label: "Enable caption",
|
||||
Description: "Ask the model for a one-line description of every image and store it as the caption.",
|
||||
Default: true,
|
||||
DecidesAction: true,
|
||||
},
|
||||
{
|
||||
Key: captionFieldKeyEnableOCR,
|
||||
Type: types.ImageFieldTypeBool,
|
||||
Label: "Enable OCR",
|
||||
Description: "Extract the text that appears in the image.",
|
||||
Default: true,
|
||||
DecidesAction: true,
|
||||
},
|
||||
// The thinking switches are action-level fields, declared once at the
|
||||
// action layer and carried in here — a pipeline does not re-declare
|
||||
// an action's tunable.
|
||||
imageActionThinkingFields[0],
|
||||
imageActionThinkingFields[1],
|
||||
}
|
||||
}
|
||||
|
||||
// Rules is what the panel has to be told: caption and OCR can each be turned
|
||||
// off by hand, and a knowledge base that turned off both would process every
|
||||
// image by doing nothing to it, so at least one has to stay on. The thinking
|
||||
// switches are deliberately absent — they tune the calls, and the actions they
|
||||
// tune run whether they are on or off.
|
||||
//
|
||||
// The pair reads as one rule and not two because "off and off" is the only
|
||||
// forbidden combination: caption off alone still leaves the image OCRed, and
|
||||
// OCR off alone still leaves it described.
|
||||
func (defaultPipeline) Rules() []types.ImagePipelineRules {
|
||||
return []types.ImagePipelineRules{{
|
||||
AtLeastOne: []string{captionFieldKeyEnableCaption, captionFieldKeyEnableOCR},
|
||||
MessageKey: "imagePipeline.errors.noActionEnabled",
|
||||
}}
|
||||
}
|
||||
|
||||
// Validate is the rule above, enforced where it cannot be argued with.
|
||||
func (defaultPipeline) Validate(params map[string]any) error {
|
||||
return firstImagePipelineError(EvaluateImagePipelineRules(
|
||||
defaultPipeline{}.ID(), defaultPipeline{}.Rules(), defaultPipeline{}.Fields(), params,
|
||||
))
|
||||
}
|
||||
|
||||
func (defaultPipeline) Run(ctx context.Context, r *runContext) error {
|
||||
// The keys the shared actions read. Set before anything runs, so that a
|
||||
// guard returning early below still leaves them filled in.
|
||||
r.captionThinkingKey = imageFieldKeyCaptionThinking
|
||||
r.ocrThinkingKey = imageFieldKeyOCRThinking
|
||||
|
||||
// Both switches belong to this pipeline alone. Neither is a whole-task
|
||||
// limit: an image handled by another pipeline may skip its caption while
|
||||
// this one still captures it, so the field names are only meaningful
|
||||
// relative to this pipeline and are recorded under those same names.
|
||||
// Switching both off is guarded in the panel; if an API caller does it
|
||||
// anyway, the run simply records that nothing was configured to happen.
|
||||
if r.BoolParam(captionFieldKeyEnableCaption) {
|
||||
if err := r.execute(ctx, types.ImageActionCaption); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !r.BoolParam(captionFieldKeyEnableOCR) {
|
||||
return nil
|
||||
}
|
||||
if err := r.execute(ctx, types.ImageActionOCR); err != nil {
|
||||
return err
|
||||
}
|
||||
// Recorded so a trace row shows which switches were actually in force.
|
||||
// Without it, a switch that read false and an action that never ran look
|
||||
// identical in the output.
|
||||
r.out["params"] = r.ParamSnapshot(captionFieldKeyEnableCaption, captionFieldKeyEnableOCR)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,496 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// stubVLM answers observation prompts with a well-formed observation (a block of
|
||||
// text, so the policy wants OCR) and every other prompt with prose. It records
|
||||
// each prompt so a test can tell how many model calls a pipeline spent.
|
||||
type stubVLM struct{ prompts []string }
|
||||
|
||||
func (s *stubVLM) Predict(_ context.Context, _ [][]byte, prompt string) (string, error) {
|
||||
s.prompts = append(s.prompts, prompt)
|
||||
if strings.Contains(prompt, "Observe the following") {
|
||||
return "contain.text: block\ncontain.data_visual: true\nDESCRIPTION: a stubbed image\n", nil
|
||||
}
|
||||
return "stubbed answer\n", nil
|
||||
}
|
||||
|
||||
// PredictWithOptions forwards to Predict: the stub records prompts, so the
|
||||
// switch the pipeline asked for is not part of what it pins.
|
||||
func (s *stubVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, prompt string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return s.Predict(context.Background(), nil, prompt)
|
||||
}
|
||||
|
||||
func (s *stubVLM) GetModelName() string { return "stub" }
|
||||
func (s *stubVLM) GetModelID() string { return "stub" }
|
||||
|
||||
var _ vlm.VLM = (*stubVLM)(nil)
|
||||
|
||||
// runOneImagePipeline drives a whole pipeline for one image, exactly as
|
||||
// processImage does, and returns the run context plus the model it talked to.
|
||||
// params are the pipeline's private tunables, as the knowledge base would have
|
||||
// resolved them; an empty map means "whatever the pipeline defaults to".
|
||||
func runOneImagePipeline(t *testing.T, attrsEnabled bool, params map[string]any) (*runContext, *stubVLM) {
|
||||
t.Helper()
|
||||
model := &stubVLM{}
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
ImageAttrsEnabled: attrsEnabled,
|
||||
ImagePipelineID: types.ImagePipelineIDFor(attrsEnabled),
|
||||
ImagePipelineParams: params,
|
||||
}
|
||||
pipeline := selectImagePipeline(payload)
|
||||
if pipeline == nil {
|
||||
t.Fatalf("no pipeline registered for attrsEnabled=%v", attrsEnabled)
|
||||
}
|
||||
r := &runContext{
|
||||
payload: payload,
|
||||
params: params,
|
||||
declared: pipeline.Fields(),
|
||||
model: model,
|
||||
imageBytes: []byte("fake image bytes"),
|
||||
vlmCfg: types.VLMConfig{},
|
||||
imageInfo: &types.ImageInfo{},
|
||||
out: types.JSONMap{},
|
||||
}
|
||||
if err := pipeline.Run(context.Background(), r); err != nil {
|
||||
t.Fatalf("pipeline %s returned an error: %v", pipeline.ID(), err)
|
||||
}
|
||||
return r, model
|
||||
}
|
||||
|
||||
// TestImagePipelineActionSequences pins how the private switches of each
|
||||
// pipeline map onto the actions that run. The sequence is the per-image answer to
|
||||
// "what did this image go through", so it is what a reader of a trace replays
|
||||
// when an image came out wrong.
|
||||
//
|
||||
// The two pipelines own different keys on purpose: the default pipeline asks
|
||||
// whether to run a step, smartocr asks whether OCR may be spent at all. Both
|
||||
// default to on, which is what an untouched knowledge base gets.
|
||||
func TestImagePipelineActionSequences(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
// attrs picks the pipeline; the other two are that pipeline's fields.
|
||||
attrs bool
|
||||
caption, ocr bool
|
||||
pipeline types.ImagePipelineID
|
||||
want []types.ImageActionID
|
||||
wantVLMCalls int
|
||||
}{
|
||||
{
|
||||
"observation, caption and OCR", true, true, true, types.ImagePipelineSmartOCR,
|
||||
[]types.ImageActionID{types.ImageActionObservationCaption, types.ImageActionOCR},
|
||||
2,
|
||||
},
|
||||
{
|
||||
"observation and caption", true, false, true, types.ImagePipelineSmartOCR,
|
||||
[]types.ImageActionID{types.ImageActionObservationCaption, types.ImageActionOCR},
|
||||
2,
|
||||
},
|
||||
{
|
||||
"observation, caption kept", true, true, false, types.ImagePipelineSmartOCR,
|
||||
[]types.ImageActionID{types.ImageActionObservationCaption},
|
||||
1,
|
||||
},
|
||||
{
|
||||
"observation, caption dropped", true, false, false, types.ImagePipelineSmartOCR,
|
||||
[]types.ImageActionID{types.ImageActionObservationCaption},
|
||||
1,
|
||||
},
|
||||
{
|
||||
"caption and OCR", false, true, true, types.ImagePipelineDefault,
|
||||
[]types.ImageActionID{types.ImageActionCaption, types.ImageActionOCR},
|
||||
2,
|
||||
},
|
||||
{
|
||||
"caption only", false, true, false, types.ImagePipelineDefault,
|
||||
[]types.ImageActionID{types.ImageActionCaption},
|
||||
1,
|
||||
},
|
||||
{
|
||||
"OCR only", false, false, true, types.ImagePipelineDefault,
|
||||
[]types.ImageActionID{types.ImageActionOCR},
|
||||
1,
|
||||
},
|
||||
{
|
||||
"nothing at all", false, false, false, types.ImagePipelineDefault,
|
||||
nil, 0,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
params := map[string]any{
|
||||
smartFieldKeyCaptureCaption: tc.caption,
|
||||
smartFieldKeyAllowOCR: tc.ocr,
|
||||
captionFieldKeyEnableCaption: tc.caption,
|
||||
captionFieldKeyEnableOCR: tc.ocr,
|
||||
}
|
||||
if !tc.attrs {
|
||||
// The observed pipeline ignores the default pipeline's keys and
|
||||
// vice versa; each row only sets the ones its pipeline declares.
|
||||
params = map[string]any{
|
||||
captionFieldKeyEnableCaption: tc.caption,
|
||||
captionFieldKeyEnableOCR: tc.ocr,
|
||||
}
|
||||
}
|
||||
r, model := runOneImagePipeline(t, tc.attrs, params)
|
||||
if got := selectImagePipeline(r.payload).ID(); got != tc.pipeline {
|
||||
t.Errorf("pipeline = %q, want %q", got, tc.pipeline)
|
||||
}
|
||||
if len(r.actions) != len(tc.want) {
|
||||
t.Fatalf("actions = %v, want %v", r.actions, tc.want)
|
||||
}
|
||||
for i, want := range tc.want {
|
||||
if r.actions[i] != want {
|
||||
t.Errorf("action[%d] = %q, want %q", i, r.actions[i], want)
|
||||
}
|
||||
}
|
||||
if len(model.prompts) != tc.wantVLMCalls {
|
||||
t.Errorf("VLM calls = %d (%v), want %d", len(model.prompts), model.prompts, tc.wantVLMCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPipelineFieldsArePrivate is the contract the settings panel rests on:
|
||||
// PIPELINE-private keys must not overlap, because the knowledge base stores
|
||||
// them flat and a collision would make one pipeline's panel control another's
|
||||
// run. ACTION-level keys ("action.key" — the thinking switches) are the
|
||||
// deliberate exception: they mean the same tunable wherever the action runs,
|
||||
// so every pipeline declares the same pair.
|
||||
func TestPipelineFieldsArePrivate(t *testing.T) {
|
||||
seen := make(map[string]types.ImagePipelineID)
|
||||
for _, spec := range ListImagePipelines() {
|
||||
keys := make(map[string]bool)
|
||||
for _, field := range spec.Fields {
|
||||
if keys[field.Key] {
|
||||
t.Errorf("pipeline %q declares field %q twice", spec.ID, field.Key)
|
||||
}
|
||||
keys[field.Key] = true
|
||||
if field.Key == "" {
|
||||
t.Errorf("pipeline %q declares a field with an empty key", spec.ID)
|
||||
continue
|
||||
}
|
||||
if field.Default == nil {
|
||||
t.Errorf("field %q of pipeline %q has no default, so the panel would show an unset toggle",
|
||||
field.Key, spec.ID)
|
||||
}
|
||||
if field.Type != types.ImageFieldTypeBool &&
|
||||
field.Type != types.ImageFieldTypeString &&
|
||||
field.Type != types.ImageFieldTypeEnum {
|
||||
t.Errorf("field %q of pipeline %q has an unknown type %q", field.Key, spec.ID, field.Type)
|
||||
}
|
||||
if other, dup := seen[field.Key]; dup && other != spec.ID {
|
||||
if field.Key == imageFieldKeyCaptionThinking || field.Key == imageFieldKeyOCRThinking {
|
||||
continue // action-level keys are shared by design
|
||||
}
|
||||
t.Errorf("field %q is declared by both %q and %q; pipeline fields must not overlap",
|
||||
field.Key, other, spec.ID)
|
||||
}
|
||||
seen[field.Key] = spec.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestListImagePipelinesMatchesRegistration pins that the endpoint's payload is
|
||||
// the registry itself: a pipeline registered but missing from the list, or
|
||||
// listed without the fields it declares, would leave the panel rendering a
|
||||
// control that the run never reads.
|
||||
func TestListImagePipelinesMatchesRegistration(t *testing.T) {
|
||||
specs := ListImagePipelines()
|
||||
if len(specs) < 2 {
|
||||
t.Fatalf("list = %d pipelines, want at least the two registered", len(specs))
|
||||
}
|
||||
for _, spec := range specs {
|
||||
registered, ok := imagePipelineRegistry[spec.ID]
|
||||
if !ok {
|
||||
t.Fatalf("list carries pipeline %q, which is not registered", spec.ID)
|
||||
}
|
||||
if spec.Name != registered.Name() {
|
||||
t.Errorf("name of %q = %q, want %q", spec.ID, spec.Name, registered.Name())
|
||||
}
|
||||
if got := len(spec.Fields); got != len(registered.Fields()) {
|
||||
t.Errorf("fields of %q = %d, want %d", spec.ID, got, len(registered.Fields()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParamFallsBackToDeclaredDefault pins that a knowledge base which never
|
||||
// set a tunable still runs with the pipeline's default rather than a zero value:
|
||||
// an unset map would otherwise silently mean "off" for every switch.
|
||||
func TestParamFallsBackToDeclaredDefault(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineDefault]
|
||||
r := &runContext{out: types.JSONMap{}, imageInfo: &types.ImageInfo{}, declared: pipeline.Fields()}
|
||||
if got := r.BoolParam(captionFieldKeyEnableOCR); got != true {
|
||||
t.Errorf("BoolParam(%q) with no params = %v, want true (the declared default)",
|
||||
captionFieldKeyEnableOCR, got)
|
||||
}
|
||||
// A key this pipeline never declared reads nil rather than panicking: an old
|
||||
// stored config may carry parameters for a pipeline that has since changed.
|
||||
if got := r.Param("no_such_field"); got != nil {
|
||||
t.Errorf("Param(\"no_such_field\") = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationPipelineDeclaresNoActionControls pins the panel contract of
|
||||
// smartocr: the observed attributes decide which work happens, so the panel
|
||||
// gets no switch that turns an action on or off. The two thinking switches are
|
||||
// the exception — they change how a call runs, not whether one does.
|
||||
func TestObservationPipelineDeclaresNoActionControls(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineSmartOCR]
|
||||
for _, field := range pipeline.Fields() {
|
||||
if field.DecidesAction {
|
||||
t.Errorf("smartocr declares field %q as an action control; "+
|
||||
"the observed attributes decide that, not the user",
|
||||
field.Key)
|
||||
}
|
||||
}
|
||||
r := &runContext{out: types.JSONMap{}, imageInfo: &types.ImageInfo{}, declared: pipeline.Fields()}
|
||||
if !r.BoolParamOr(smartFieldKeyAllowOCR, true) {
|
||||
t.Error("allow_ocr with no params must default to true")
|
||||
}
|
||||
r.params = map[string]any{smartFieldKeyAllowOCR: false}
|
||||
if r.BoolParamOr(smartFieldKeyAllowOCR, true) {
|
||||
t.Error("an explicit allow_ocr=false must override the pinned default")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPipelineRulesMatchTheirFields pins that a rule names fields the pipeline
|
||||
// actually offers, and that it names action controls rather than switches that
|
||||
// only tune a call: a rule over a field that runs anyway would never be broken,
|
||||
// and a rule naming a field the panel does not render could not be fixed by
|
||||
// the user who has to fix it.
|
||||
func TestPipelineRulesMatchTheirFields(t *testing.T) {
|
||||
for _, id := range []types.ImagePipelineID{types.ImagePipelineDefault, types.ImagePipelineSmartOCR} {
|
||||
pipeline := imagePipelineRegistry[id]
|
||||
for _, rule := range pipeline.Rules() {
|
||||
if rule.MessageKey == "" {
|
||||
t.Errorf("%s rule over %v carries no message key, so the panel could only say that something is wrong",
|
||||
id, rule.AtLeastOne)
|
||||
}
|
||||
for _, key := range rule.AtLeastOne {
|
||||
var field types.ImageFieldDef
|
||||
found := false
|
||||
for _, candidate := range pipeline.Fields() {
|
||||
if candidate.Key == key {
|
||||
field, found = candidate, true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s rule names field %q, which this pipeline does not declare", id, key)
|
||||
continue
|
||||
}
|
||||
if field.DecidesAction {
|
||||
continue
|
||||
}
|
||||
t.Errorf("%s rule names field %q, which only tunes a call that runs either way", id, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualPipelineDeclaresRules pins the other half of the same contract: the
|
||||
// pipeline that leaves every action to the operator's switches must say so, or
|
||||
// the panel would never warn that a knowledge base is set to process images by
|
||||
// running nothing. The two halves together are what keep the "at least one
|
||||
// action has to stay on" check off the smart pipeline, whose actions it cannot
|
||||
// switch off, and on the manual one, where they are the user's to decide.
|
||||
func TestManualPipelineDeclaresRules(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineDefault]
|
||||
if len(pipeline.Rules()) == 0 {
|
||||
t.Fatal("the manual pipeline declares no rule, so a knowledge base " +
|
||||
"set to run nothing would save without warning")
|
||||
}
|
||||
// A thinking switch tunes the call; like the smart pipeline's, it may be off
|
||||
// for every image without leaving anything unprocessed.
|
||||
for _, field := range pipeline.Fields() {
|
||||
if strings.Contains(field.Key, "thinking") && field.DecidesAction {
|
||||
t.Errorf("thinking switch %q must not decide an action; the call runs either way", field.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSmartPipelineDeclaresNoRules pins that the pipeline which schedules its
|
||||
// own actions declares no rule at all — the observed attributes decide the work,
|
||||
// so no combination of the switches it offers can be invalid, and the panel has
|
||||
// nothing to warn about.
|
||||
func TestSmartPipelineDeclaresNoRules(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineSmartOCR]
|
||||
if len(pipeline.Rules()) != 0 {
|
||||
t.Errorf("smartocr declares %d rules; it switches its own actions on and off", len(pipeline.Rules()))
|
||||
}
|
||||
if err := pipeline.Validate(map[string]any{
|
||||
imageFieldKeyCaptionThinking: false,
|
||||
imageFieldKeyOCRThinking: false,
|
||||
}); err != nil {
|
||||
t.Errorf("smartocr with both thinking switches off = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownPipelineFieldIsIgnored keeps a stale stored parameter harmless. A
|
||||
// knowledge base may carry a key the running pipeline no longer declares; it
|
||||
// must not leak into the trace snapshot nor change the run.
|
||||
func TestUnknownPipelineFieldIsIgnored(t *testing.T) {
|
||||
r, model := runOneImagePipeline(t, false, map[string]any{
|
||||
captionFieldKeyEnableCaption: true,
|
||||
captionFieldKeyEnableOCR: true,
|
||||
"removed_by_a_later_release": true,
|
||||
})
|
||||
if len(model.prompts) != 2 {
|
||||
t.Fatalf("VLM calls = %d (%v), want 2", len(model.prompts), model.prompts)
|
||||
}
|
||||
if _, leaked := r.out["params"].(types.JSONMap)["removed_by_a_later_release"]; leaked {
|
||||
t.Error("an undeclared key must not reach the trace snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationCaptionSharesOneCall is the point of the whole decomposition:
|
||||
// with attributes on, describing and observing is one action and therefore one
|
||||
// model request, and the standalone caption action is never scheduled.
|
||||
func TestObservationCaptionSharesOneCall(t *testing.T) {
|
||||
r, model := runOneImagePipeline(t, true, map[string]any{
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
smartFieldKeyAllowOCR: false,
|
||||
})
|
||||
if len(model.prompts) != 1 {
|
||||
t.Fatalf("VLM calls = %d (%v), want 1 — observation and caption must share a request",
|
||||
len(model.prompts), model.prompts)
|
||||
}
|
||||
if !strings.Contains(model.prompts[0], "Observe the following") {
|
||||
t.Errorf("the single prompt is not the observation prompt: %q", model.prompts[0])
|
||||
}
|
||||
for _, id := range r.actions {
|
||||
if id == types.ImageActionCaption {
|
||||
t.Error("the standalone caption action ran; observation must carry the caption")
|
||||
}
|
||||
}
|
||||
// The shared request also fills the caption slot, which is the other half of
|
||||
// "observation carries caption".
|
||||
if got := r.imageInfo.Caption; got != "a stubbed image" {
|
||||
t.Errorf("Caption = %q, want the description from the same call", got)
|
||||
}
|
||||
if _, ok := r.out["attr_policy"]; !ok {
|
||||
t.Error(`out["attr_policy"] is missing; the trace must state the decision`)
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationPipelineTrace pins what a trace row says when OCR is skipped by
|
||||
// the attribute policy, which is the case image_info alone cannot explain.
|
||||
func TestObservationPipelineTrace(t *testing.T) {
|
||||
r, _ := runOneImagePipeline(t, true, map[string]any{
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
smartFieldKeyAllowOCR: false,
|
||||
})
|
||||
if got := r.out["ocr_skipped"]; got != "attr_policy" {
|
||||
t.Errorf(`out["ocr_skipped"] = %v, want "attr_policy"`, got)
|
||||
}
|
||||
if _, ok := r.out["ocr_text"]; ok {
|
||||
t.Error("no OCR action ran, so no ocr slot may exist")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultPipelineTrace pins the plain path: both actions run and no policy
|
||||
// field is invented, because there is nothing to decide from.
|
||||
func TestDefaultPipelineTrace(t *testing.T) {
|
||||
r, _ := runOneImagePipeline(t, false, map[string]any{
|
||||
captionFieldKeyEnableCaption: true,
|
||||
captionFieldKeyEnableOCR: true,
|
||||
})
|
||||
if r.imageInfo.Caption == "" {
|
||||
t.Error("caption slot is empty after the caption action")
|
||||
}
|
||||
if r.imageInfo.OCRText == "" {
|
||||
t.Error("OCR slot is empty after the OCR action")
|
||||
}
|
||||
if _, ok := r.out["attr_policy"]; ok {
|
||||
t.Error("caption+OCR mode has no attributes, so it must not record a policy")
|
||||
}
|
||||
if _, ok := r.out["ocr_skipped"]; ok {
|
||||
t.Errorf("unexpected ocr_skipped in the plain path: %v", r.out["ocr_skipped"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImagePipelineIDsAreStable pins the two ids and the strings that reach the
|
||||
// trace — the labels a reader greps for.
|
||||
func TestImagePipelineIDsAreStable(t *testing.T) {
|
||||
if got := types.ImagePipelineIDFor(true); got != types.ImagePipelineSmartOCR {
|
||||
t.Errorf("ImagePipelineIDFor(true) = %q", got)
|
||||
}
|
||||
if got := string(types.ImagePipelineSmartOCR); got != "smartocr" {
|
||||
t.Errorf(`pipeline id = %q, want "smartocr"`, got)
|
||||
}
|
||||
if got := string(types.ImagePipelineDefault); got != "default" {
|
||||
t.Errorf(`pipeline id = %q, want "default"`, got)
|
||||
}
|
||||
if got := string(types.NormalizeImagePipelineID(types.ImagePipelineLegacyCaptionOCR)); got != "default" {
|
||||
t.Errorf(`legacy id normalized = %q, want "default"`, got)
|
||||
}
|
||||
if got := string(types.NormalizeImagePipelineID(types.ImagePipelineLegacyObCapOCR)); got != "smartocr" {
|
||||
t.Errorf(`legacy id normalized = %q, want "smartocr"`, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageActionRegistryCoversSharedActions makes sure every action declared in
|
||||
// types is actually runnable: a declaration without a binding would surface as a
|
||||
// silently skipped field instead of a build error.
|
||||
func TestImageActionRegistryCoversSharedActions(t *testing.T) {
|
||||
if len(types.SharedImageActions) == 0 {
|
||||
t.Fatal("SharedImageActions is empty")
|
||||
}
|
||||
for _, spec := range types.SharedImageActions {
|
||||
registered, ok := imageActionRegistry[spec.ID]
|
||||
if !ok {
|
||||
t.Errorf("action %q is declared but never registered", spec.ID)
|
||||
continue
|
||||
}
|
||||
if registered.handler == nil {
|
||||
t.Errorf("action %q is registered without a handler", spec.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecuteSkipsUnimplementedAction keeps the registry's failure mode honest:
|
||||
// one missing action costs one field, it does not abort the image.
|
||||
func TestExecuteSkipsUnimplementedAction(t *testing.T) {
|
||||
r := &runContext{out: types.JSONMap{}, imageInfo: &types.ImageInfo{}}
|
||||
err := r.execute(context.Background(), types.ImageActionID("not_registered"))
|
||||
if err != nil {
|
||||
t.Fatalf("execute returned %v, want nil", err)
|
||||
}
|
||||
if got := r.out["action_skipped"]; got != "not_registered" {
|
||||
t.Errorf(`out["action_skipped"] = %v, want "not_registered"`, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParamPrefersPipelineNamespace pins the future-proofing in Param: a
|
||||
// pipeline-private key stored under the running pipeline's namespace
|
||||
// ("<pipeline>.<key>") wins over a bare key, and another pipeline never reads
|
||||
// a namespace that is not its own. Writes are bare today — no two pipelines
|
||||
// declare the same private key — so this only guards the day one appears.
|
||||
func TestParamPrefersPipelineNamespace(t *testing.T) {
|
||||
r := &runContext{
|
||||
out: types.JSONMap{},
|
||||
imageInfo: &types.ImageInfo{},
|
||||
pipelineID: "smartocr",
|
||||
params: map[string]any{
|
||||
"smartocr.allow_ocr": false,
|
||||
"allow_ocr": true,
|
||||
},
|
||||
}
|
||||
if got := r.BoolParamOr(smartFieldKeyAllowOCR, true); got != false {
|
||||
t.Errorf("the namespaced key lost to the bare legacy key: allow_ocr = %v, want false", got)
|
||||
}
|
||||
r.pipelineID = "default"
|
||||
if got := r.BoolParamOr(smartFieldKeyAllowOCR, true); got != true {
|
||||
t.Errorf("another pipeline read smartocr's namespace: allow_ocr = %v, want true (bare legacy)", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestResolveProcessConfig_LegacyOverrideSelectsImagePipeline(t *testing.T) {
|
||||
for _, base := range []bool{false, true} {
|
||||
for _, explicitKB := range []bool{false, true} {
|
||||
kb := &types.KnowledgeBase{ImageProcessingConfig: types.ImageProcessingConfig{
|
||||
ImageAttrsEnabled: base,
|
||||
ImagePipelineParams: map[string]any{"caption.thinking": true},
|
||||
}}
|
||||
if explicitKB {
|
||||
kb.ImageProcessingConfig.ImagePipelineID = string(types.ImagePipelineIDFor(base))
|
||||
}
|
||||
for _, override := range []bool{false, true} {
|
||||
eff := ResolveProcessConfig(kb, &types.KnowledgeProcessOverrides{ImageAttrsEnabled: &override})
|
||||
pipeline := selectImagePipeline(&types.ImageMultimodalPayload{
|
||||
ImagePipelineID: eff.ImagePipelineID, ImageAttrsEnabled: eff.ImageAttrsEnabled,
|
||||
})
|
||||
require.Equal(t, types.ImagePipelineIDFor(override), pipeline.ID())
|
||||
if base == override {
|
||||
require.Equal(t, kb.ImageProcessingConfig.ImagePipelineParams, eff.ImagePipelineParams)
|
||||
} else {
|
||||
require.Empty(t, eff.ImagePipelineParams,
|
||||
"a different pipeline must not inherit the old parameters")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// A new client's explicit choice takes precedence over its legacy switch.
|
||||
id := types.ImagePipelineLegacyCaptionOCR
|
||||
eff := ResolveProcessConfig(&types.KnowledgeBase{}, &types.KnowledgeProcessOverrides{
|
||||
ImageAttrsEnabled: processConfigBoolPtr(true), ImagePipelineID: &id,
|
||||
ImagePipelineParams: map[string]any{"enable_caption": false},
|
||||
})
|
||||
require.Equal(t, types.ImagePipelineDefault, eff.ImagePipelineID)
|
||||
require.False(t, eff.ImageAttrsEnabled)
|
||||
require.Equal(t, false, eff.ImagePipelineParams["enable_caption"])
|
||||
}
|
||||
|
||||
func TestProcessOverrides_ValidateImagePipelineBeforePersistence(t *testing.T) {
|
||||
id := types.ImagePipelineDefault
|
||||
for _, enabled := range []bool{false, true} {
|
||||
for _, caption := range []bool{false, true} {
|
||||
kb := &types.KnowledgeBase{}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
EnableMultimodel: &enabled, ImagePipelineID: &id,
|
||||
ImagePipelineParams: map[string]any{"enable_caption": caption, "enable_ocr": false},
|
||||
}
|
||||
knowledge := &types.Knowledge{}
|
||||
_, uploadErr := resolveFileImportProcessConfig(context.Background(), kb, "pdf", overrides, nil)
|
||||
_, reparseErr := ApplyKnowledgeProcessOverrides(
|
||||
context.Background(), kb, knowledge, overrides, []string{"pdf"}, nil)
|
||||
validationErr := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"pdf"})
|
||||
for _, err := range []error{uploadErr, reparseErr, validationErr} {
|
||||
if enabled && !caption {
|
||||
var badRequest *werrors.AppError
|
||||
require.ErrorAs(t, err, &badRequest)
|
||||
require.Equal(t, werrors.ErrBadRequest, badRequest.Code)
|
||||
require.Contains(t, err.Error(), "noActionEnabled")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
if enabled && !caption {
|
||||
got, err := knowledge.ProcessOverrides()
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, got, "invalid reparse settings must not be persisted")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessOverrides_ValidateLegacyMultimodalFlag(t *testing.T) {
|
||||
id := types.ImagePipelineDefault
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ImagePipelineID: &id,
|
||||
ImagePipelineParams: map[string]any{"enable_caption": false, "enable_ocr": false},
|
||||
}
|
||||
enabled := true
|
||||
_, err := resolveFileImportProcessConfig(context.Background(), &types.KnowledgeBase{}, "pdf", overrides, &enabled)
|
||||
require.ErrorContains(t, err, "noActionEnabled")
|
||||
_, err = ApplyKnowledgeProcessOverrides(
|
||||
context.Background(), &types.KnowledgeBase{}, &types.Knowledge{}, overrides, []string{"pdf"}, &enabled)
|
||||
require.ErrorContains(t, err, "noActionEnabled")
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
// The rules a pipeline declares over its own tunables, evaluated from the
|
||||
// values a knowledge base resolved. The panel requests this evaluation so it
|
||||
// can warn before the user commits, and the save path runs it
|
||||
// again on the backend so that a caller which never showed the panel cannot
|
||||
// store something the pipeline would not run. Both sides read the same rules
|
||||
// out of the spec, so a rule added here shows up in the UI with no frontend
|
||||
// change.
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// EvaluateImagePipelineRules walks a pipeline's rules and reports every one its
|
||||
// tunables break, in declaration order. Reporting all of them rather than the
|
||||
// first is the point: each rule carries its own wording and its own field, so a
|
||||
// panel that shows three broken rules tells the user exactly what to change
|
||||
// instead of making them fix one at a time and re-read the page. An empty
|
||||
// result is a configuration the pipeline will run.
|
||||
//
|
||||
// A rule names the fields it is about rather than a condition to compute:
|
||||
// "at least one of these is on" is what a set of action switches can mean, and
|
||||
// a pipeline that offers none to switch off declares no rule at all rather than
|
||||
// one that always holds.
|
||||
//
|
||||
// A field the parameters do not mention reads its declared default, the same
|
||||
// way the run does: a knowledge base saved before a field existed is not told
|
||||
// that it switched something off by never mentioning it.
|
||||
func EvaluateImagePipelineRules(
|
||||
pipelineID types.ImagePipelineID,
|
||||
rules []types.ImagePipelineRules,
|
||||
fields []types.ImageFieldDef,
|
||||
params map[string]any,
|
||||
) []*types.ImagePipelineValidationError {
|
||||
// Use the execution-time resolver for namespace precedence, defaults and
|
||||
// boolean coercion. A separate interpretation can accept a no-op pipeline.
|
||||
r := &runContext{pipelineID: string(pipelineID), declared: fields, params: params}
|
||||
broken := make([]*types.ImagePipelineValidationError, 0, len(rules))
|
||||
for _, rule := range rules {
|
||||
if len(rule.AtLeastOne) == 0 || rule.MessageKey == "" {
|
||||
continue
|
||||
}
|
||||
satisfied := false
|
||||
for _, key := range rule.AtLeastOne {
|
||||
if r.BoolParam(key) {
|
||||
satisfied = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if satisfied {
|
||||
continue
|
||||
}
|
||||
field := rule.Field
|
||||
if field == "" {
|
||||
field = rule.AtLeastOne[0]
|
||||
}
|
||||
broken = append(broken, &types.ImagePipelineValidationError{
|
||||
MessageKey: rule.MessageKey,
|
||||
Field: field,
|
||||
})
|
||||
}
|
||||
return broken
|
||||
}
|
||||
|
||||
// firstImagePipelineError turns a list of broken rules into the error one save
|
||||
// path reports. A rejection can only say what is wrong about the whole request,
|
||||
// so the first broken rule stands for all of them.
|
||||
func firstImagePipelineError(broken []*types.ImagePipelineValidationError) error {
|
||||
if len(broken) == 0 {
|
||||
return nil
|
||||
}
|
||||
return broken[0]
|
||||
}
|
||||
|
||||
// validateImagePipelineConfig is the save path's own check, run after the
|
||||
// settings panel has already asked and been answered. It is here so that a
|
||||
// caller which never showed a panel — an API client, an import, a script —
|
||||
// cannot store a combination the pipeline would not run either.
|
||||
//
|
||||
// The pipeline id is resolved the way the run resolves it rather than taken
|
||||
// from the field as written: a knowledge base saved before the id field existed
|
||||
// carries none, and that must not make the check a no-op.
|
||||
func (s *knowledgeBaseService) validateImagePipelineConfig(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
) error {
|
||||
cfg := kb.ImageProcessingConfig
|
||||
pipelineID := types.ResolveImagePipelineID(&cfg)
|
||||
err := ValidateImagePipelineParams(pipelineID, cfg.ImagePipelineParams)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[kb.image] rejecting pipeline %q params for knowledge base %q: %v",
|
||||
pipelineID, kb.ID, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestManualPipelineRejectsEveryActionOff covers the rule the manual pipeline
|
||||
// declares: with its two action switches off there is nothing left to do to an
|
||||
// image, and the run would record that rather than complain, so the rejection
|
||||
// has to come from the check before it is stored.
|
||||
func TestManualPipelineRejectsEveryActionOff(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineDefault]
|
||||
err := pipeline.Validate(map[string]any{
|
||||
captionFieldKeyEnableCaption: false,
|
||||
captionFieldKeyEnableOCR: false,
|
||||
})
|
||||
invalid, ok := err.(*types.ImagePipelineValidationError)
|
||||
if !ok {
|
||||
t.Fatalf("Validate(manual, both off) = %v, want an ImagePipelineValidationError", err)
|
||||
}
|
||||
if invalid.MessageKey == "" {
|
||||
t.Error("the rejection carries no message key, so the panel could not translate it")
|
||||
}
|
||||
// Pointing at a switch is what tells the user which one to turn back on.
|
||||
if invalid.Field != captionFieldKeyEnableCaption {
|
||||
t.Errorf("field = %q, want %q (the first of the pair)", invalid.Field, captionFieldKeyEnableCaption)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualPipelineAcceptsOneActionOn is the other half: a knowledge base may
|
||||
// run captions only, or OCR only, and a thinking switch that nobody touched must
|
||||
// not be read as "off, therefore nothing to do".
|
||||
func TestManualPipelineAcceptsOneActionOn(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineDefault]
|
||||
cases := []map[string]any{
|
||||
{captionFieldKeyEnableCaption: true, captionFieldKeyEnableOCR: false},
|
||||
{captionFieldKeyEnableCaption: false, captionFieldKeyEnableOCR: true},
|
||||
// Neither key carried: both default to true, which is what a knowledge
|
||||
// base saved before these fields existed gets.
|
||||
{},
|
||||
// A thinking switch off changes nothing about whether the actions run.
|
||||
{
|
||||
captionFieldKeyEnableCaption: true, captionFieldKeyEnableOCR: true,
|
||||
imageFieldKeyCaptionThinking: false, imageFieldKeyOCRThinking: false,
|
||||
},
|
||||
// Values that reached the pipeline through a form may arrive as strings.
|
||||
{captionFieldKeyEnableCaption: "true", captionFieldKeyEnableOCR: "false"},
|
||||
}
|
||||
for i, params := range cases {
|
||||
if err := pipeline.Validate(params); err != nil {
|
||||
t.Errorf("case %d (%v) = %v, want nil", i, params, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualPipelineRejectsOffByValue pins that only the value decides, not the
|
||||
// spelling: a false written as a string, or a zero, has to fail the same rule a
|
||||
// boolean false does, or a caller posting JSON by hand slips past the check.
|
||||
func TestManualPipelineRejectsOffByValue(t *testing.T) {
|
||||
pipeline := imagePipelineRegistry[types.ImagePipelineDefault]
|
||||
cases := map[string]any{
|
||||
"boolean": false,
|
||||
"string": "false",
|
||||
"zero int": 0,
|
||||
}
|
||||
for _, caption := range cases {
|
||||
for _, ocr := range cases {
|
||||
err := pipeline.Validate(map[string]any{
|
||||
captionFieldKeyEnableCaption: caption,
|
||||
captionFieldKeyEnableOCR: ocr,
|
||||
})
|
||||
if err == nil {
|
||||
t.Errorf("both off as %#v and %#v = nil, want the rule to fire", caption, ocr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownPipelineIDSkipsValidation keeps a stored id this build does not
|
||||
// ship out of blocking the save. The run falls back the same way, so a
|
||||
// configuration that can no longer be reproduced must still be editable.
|
||||
func TestUnknownPipelineIDSkipsValidation(t *testing.T) {
|
||||
err := ValidateImagePipelineParams(types.ImagePipelineID("pipeline_from_a_future_release"), map[string]any{})
|
||||
if err != nil {
|
||||
t.Errorf("Validate(unknown id) = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidationGoesThroughTheSavedPipeline pins that the check resolves the id
|
||||
// the way the run does: a knowledge base that never named a pipeline is asking
|
||||
// for the manual one, and its switches are still the ones under test.
|
||||
func TestValidationGoesThroughTheSavedPipeline(t *testing.T) {
|
||||
if err := ValidateImagePipelineParams("", nil); err != nil {
|
||||
t.Errorf("Validate(\"\") = %v, want nil (both actions default on)", err)
|
||||
}
|
||||
err := ValidateImagePipelineParams(types.ImagePipelineDefault, map[string]any{
|
||||
captionFieldKeyEnableCaption: false,
|
||||
captionFieldKeyEnableOCR: false,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "noActionEnabled") {
|
||||
t.Errorf("Validate(manual, both off) = %v, want the noActionEnabled message key", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the public validation path and the actual model calls, including
|
||||
// legacy IDs, so accepted configurations cannot silently execute no actions.
|
||||
func TestImagePipelineValidationMatchesExecution(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
params map[string]any
|
||||
wantCalls int
|
||||
}{
|
||||
{"defaults", nil, 2},
|
||||
{
|
||||
"namespace overrides bare keys",
|
||||
map[string]any{
|
||||
"enable_caption": true, "enable_ocr": true,
|
||||
"default.enable_caption": false, "default.enable_ocr": false,
|
||||
},
|
||||
0,
|
||||
},
|
||||
{"namespace only", map[string]any{"default.enable_caption": false, "default.enable_ocr": false}, 0},
|
||||
{
|
||||
"namespace enables caption",
|
||||
map[string]any{
|
||||
"enable_caption": false, "enable_ocr": false,
|
||||
"default.enable_caption": true,
|
||||
},
|
||||
1,
|
||||
},
|
||||
{"other namespace ignored", map[string]any{"smartocr.enable_caption": false, "smartocr.enable_ocr": false}, 2},
|
||||
{"null uses declared default", map[string]any{"enable_caption": nil, "enable_ocr": false}, 1},
|
||||
{
|
||||
"null namespace falls through to bare key",
|
||||
map[string]any{
|
||||
"default.enable_caption": nil, "enable_caption": false,
|
||||
"enable_ocr": false,
|
||||
},
|
||||
0,
|
||||
},
|
||||
{"string true", map[string]any{"enable_caption": "true", "enable_ocr": false}, 1},
|
||||
{"string on", map[string]any{"enable_caption": "on", "enable_ocr": false}, 1},
|
||||
{"string one", map[string]any{"enable_caption": "1", "enable_ocr": false}, 1},
|
||||
{"string false", map[string]any{"enable_caption": "false", "enable_ocr": false}, 0},
|
||||
{
|
||||
"numeric values are not boolean switches",
|
||||
map[string]any{"enable_caption": float64(1), "enable_ocr": false},
|
||||
0,
|
||||
},
|
||||
}
|
||||
for _, id := range []types.ImagePipelineID{types.ImagePipelineDefault, types.ImagePipelineLegacyCaptionOCR, ""} {
|
||||
for _, tc := range cases {
|
||||
t.Run(string(id)+"/"+tc.name, func(t *testing.T) {
|
||||
enabled := true
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
EnableMultimodel: &enabled, ImagePipelineID: &id,
|
||||
ImagePipelineParams: tc.params,
|
||||
}
|
||||
validationErr := ValidateProcessOverrides(
|
||||
context.Background(), &types.KnowledgeBase{}, overrides, []string{"pdf"})
|
||||
pipelineErr := defaultPipeline{}.Validate(tc.params)
|
||||
if tc.wantCalls == 0 {
|
||||
require.ErrorContains(t, validationErr, "noActionEnabled")
|
||||
require.ErrorContains(t, pipelineErr, "noActionEnabled")
|
||||
return
|
||||
}
|
||||
require.NoError(t, validationErr)
|
||||
require.NoError(t, pipelineErr)
|
||||
repo := &attrsChunkRepo{}
|
||||
svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
|
||||
model := &stubVLM{}
|
||||
out := types.JSONMap{}
|
||||
err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
|
||||
ImageURL: "local://test.png", ImagePipelineID: id, ImagePipelineParams: tc.params,
|
||||
}, model, types.VLMConfig{}, noopSpanTracker{}, out)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, model.prompts, tc.wantCalls)
|
||||
require.Len(t, repo.created, tc.wantCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func init() { registerImagePipeline(smartOcrPipeline{}) }
|
||||
|
||||
// smartOcrPipeline observes first, then decides what to spend next. The id
|
||||
// promises that observation comes before the decision, not that caption and OCR
|
||||
// are the only things that can happen afterwards — a refine or validate action
|
||||
// between the two does not make this name a lie.
|
||||
type smartOcrPipeline struct{}
|
||||
|
||||
func (smartOcrPipeline) ID() types.ImagePipelineID { return types.ImagePipelineSmartOCR }
|
||||
|
||||
func (smartOcrPipeline) Name() string { return "Observe, caption, then decide on OCR" }
|
||||
|
||||
func (smartOcrPipeline) Description() string {
|
||||
return "Observe the image's attributes and describe it first, then decide " +
|
||||
"from the attributes whether OCR is worth a pass, saving model calls."
|
||||
}
|
||||
|
||||
// Field keys of this pipeline. allow_ocr and capture_caption are
|
||||
// PIPELINE-private keys (this pipeline answers "may OCR be spent at all",
|
||||
// which is a ceiling on the policy rather than a step to run) and deliberately
|
||||
// do not reuse the default pipeline's step switches. The thinking switches are
|
||||
// ACTION-level keys (caption.thinking / ocr.thinking) shared with the default
|
||||
// pipeline: the same action means the same tunable wherever it runs, so there
|
||||
// is nothing to isolate and no reason to invent per-pipeline names.
|
||||
const (
|
||||
smartFieldKeyAllowOCR = "allow_ocr"
|
||||
smartFieldKeyCaptureCaption = "capture_caption"
|
||||
)
|
||||
|
||||
// Fields declares this pipeline's panel controls. The model decides from the
|
||||
// observed attributes whether OCR is worth a second pass, and the caption the
|
||||
// observation produced is always kept, so there is no ceiling and no caption
|
||||
// switch to expose. Only the two thinking switches are offered: they change
|
||||
// how the two model calls work, not which ones happen.
|
||||
//
|
||||
// The caption key here is the same one the manual pipeline renders, and it
|
||||
// drives the same action — this pipeline reaches the caption through the
|
||||
// observation call rather than through a caption action of its own. A knowledge
|
||||
// base that never touched these switches still behaves as it always did.
|
||||
func (smartOcrPipeline) Fields() []types.ImageFieldDef {
|
||||
return []types.ImageFieldDef{
|
||||
// The thinking switches are action-level fields, declared once at the
|
||||
// action layer and carried in here.
|
||||
imageActionThinkingFields[0],
|
||||
imageActionThinkingFields[1],
|
||||
}
|
||||
}
|
||||
|
||||
// Rules returns nothing, and that is the whole answer to "what may not this
|
||||
// pipeline be set to?": observation, description and OCR are scheduled by the
|
||||
// attributes the model observed, not by the user, so no combination of the two
|
||||
// switches it offers can leave an image untouched. The panel, which evaluates
|
||||
// rules generically, therefore never warns here.
|
||||
func (smartOcrPipeline) Rules() []types.ImagePipelineRules {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate accepts everything: see Rules above.
|
||||
func (smartOcrPipeline) Validate(_ map[string]any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (smartOcrPipeline) Run(ctx context.Context, r *runContext) error {
|
||||
// The keys the shared actions read, set before the observation runs so the
|
||||
// OCR branch below leaves them filled in either way. The thinking switches
|
||||
// are action-level keys shared with the default pipeline.
|
||||
r.captionThinkingKey = imageFieldKeyCaptionThinking
|
||||
r.ocrThinkingKey = imageFieldKeyOCRThinking
|
||||
|
||||
// One request fills both the attribute slots and the caption slot, so there
|
||||
// is no separate caption action to schedule here. The allow_ocr and
|
||||
// capture_caption ceilings are deliberately NOT recorded on the trace: the
|
||||
// panel never declares them (only the two thinking switches), so they are
|
||||
// always their defaults — recording them read as if a decision had been
|
||||
// made ("allow_ocr=true" next to a declined OCR). The decision and the
|
||||
// inputs that actually drove it are recorded on attr_policy below.
|
||||
if err := r.execute(ctx, types.ImageActionObservationCaption); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The policy reads the attributes the action above just wrote; the ceiling
|
||||
// only says whether OCR is allowed at all. Both lines are recorded even
|
||||
// when OCR is skipped, because the trace has to show the decision that was
|
||||
// made rather than an OCR chunk that is simply missing. ocr_on_unobserved
|
||||
// travels with the decision: it is the clause that decides when the
|
||||
// observation failed or left attributes unanswered, and it is what tells
|
||||
// "declined on data nobody produced" apart from a confident negative.
|
||||
want := r.BoolParamOr(smartFieldKeyAllowOCR, true) &&
|
||||
DecideOCR(r.imageInfo.Attrs, r.payload.ImageActions)
|
||||
r.out["attr_policy"] = types.JSONMap{
|
||||
"ocr": want,
|
||||
"ocr_on_unobserved": r.payload.ImageActions.OCR.OnUnobserved,
|
||||
}
|
||||
r.out["image_attrs"] = r.imageInfo.Attrs.Attrs
|
||||
if !want {
|
||||
// WHY the policy said no: a failed observation left no attributes to
|
||||
// decide on (the OnUnobserved clause declined), which is a different
|
||||
// story from attributes that were observed and voted against OCR. The
|
||||
// trace keeps the two apart instead of a generic "attr_policy" that
|
||||
// reads as if a decision was made on data nobody produced.
|
||||
observationFailed := r.out["observation_failed"] == true || r.out["caption_error"] != nil
|
||||
if observationFailed {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Skipping OCR for %s (observation failed, attrs=%v)",
|
||||
r.payload.ImageURL, r.imageInfo.Attrs.Attrs)
|
||||
r.out["ocr_skipped"] = "no_ocr_after_observation_failed"
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Skipping OCR for %s (attrs=%v)",
|
||||
r.payload.ImageURL, r.imageInfo.Attrs.Attrs)
|
||||
r.out["ocr_skipped"] = "attr_policy"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return r.execute(ctx, types.ImageActionOCR)
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// thinkingCall is one model request made by a pipeline, with the thinking
|
||||
// switch the caller asked for — the value the remote model actually receives.
|
||||
type thinkingCall struct {
|
||||
prompt string
|
||||
thinking bool
|
||||
}
|
||||
|
||||
// thinkingStubVLM records the switch alongside the prompt so a test can pin
|
||||
// "what did the pipeline ask the model for", not just how many times it asked.
|
||||
type thinkingStubVLM struct{ calls []thinkingCall }
|
||||
|
||||
func (s *thinkingStubVLM) record(prompt string, thinking *bool) string {
|
||||
var on bool
|
||||
if thinking != nil {
|
||||
on = *thinking
|
||||
}
|
||||
s.calls = append(s.calls, thinkingCall{prompt: prompt, thinking: on})
|
||||
if strings.Contains(prompt, "Observe the following") {
|
||||
return "contain.text: block\ncontain.data_visual: true\nDESCRIPTION: a stubbed image\n"
|
||||
}
|
||||
return "stubbed answer\n"
|
||||
}
|
||||
|
||||
func (s *thinkingStubVLM) Predict(_ context.Context, _ [][]byte, prompt string) (string, error) {
|
||||
return s.record(prompt, nil), nil
|
||||
}
|
||||
|
||||
func (s *thinkingStubVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, prompt string, opts *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
var on *bool
|
||||
if opts != nil {
|
||||
on = opts.Thinking
|
||||
}
|
||||
return s.record(prompt, on), nil
|
||||
}
|
||||
|
||||
func (s *thinkingStubVLM) GetModelName() string { return "thinking-stub" }
|
||||
func (s *thinkingStubVLM) GetModelID() string { return "thinking-stub" }
|
||||
|
||||
var _ vlm.VLM = (*thinkingStubVLM)(nil)
|
||||
|
||||
// runThinkingPipeline drives one pipeline run against a recording stub. params
|
||||
// are the pipeline's private tunables; an empty map means "defaults".
|
||||
func runThinkingPipeline(t *testing.T, attrsEnabled bool, params map[string]any) (*runContext, *thinkingStubVLM) {
|
||||
t.Helper()
|
||||
model := &thinkingStubVLM{}
|
||||
payload := &types.ImageMultimodalPayload{
|
||||
ImageAttrsEnabled: attrsEnabled,
|
||||
ImagePipelineID: types.ImagePipelineIDFor(attrsEnabled),
|
||||
ImagePipelineParams: params,
|
||||
}
|
||||
pipeline := selectImagePipeline(payload)
|
||||
if pipeline == nil {
|
||||
t.Fatalf("no pipeline registered for attrsEnabled=%v", attrsEnabled)
|
||||
}
|
||||
r := &runContext{
|
||||
payload: payload,
|
||||
params: params,
|
||||
declared: pipeline.Fields(),
|
||||
model: model,
|
||||
imageBytes: []byte("fake image bytes"),
|
||||
vlmCfg: types.VLMConfig{},
|
||||
imageInfo: &types.ImageInfo{},
|
||||
out: types.JSONMap{},
|
||||
}
|
||||
if err := pipeline.Run(context.Background(), r); err != nil {
|
||||
t.Fatalf("pipeline %s returned an error: %v", pipeline.ID(), err)
|
||||
}
|
||||
return r, model
|
||||
}
|
||||
|
||||
// TestThinkingDefaultsToOff pins the decision that an untouched knowledge base
|
||||
// runs the cheaper, sturdier call: a quantized reasoning model that thinks its
|
||||
// way out of a short prompt will often emit nothing but reasoning and hand back
|
||||
// an empty OCR slot, which is worse than not turning thinking on at all. So
|
||||
// thinking is off unless somebody asked for it, per action.
|
||||
func TestThinkingDefaultsToOff(t *testing.T) {
|
||||
r, model := runThinkingPipeline(t, false, map[string]any{
|
||||
captionFieldKeyEnableCaption: true,
|
||||
captionFieldKeyEnableOCR: true,
|
||||
})
|
||||
if len(model.calls) != 2 {
|
||||
t.Fatalf("VLM calls = %d (%v), want 2", len(model.calls), model.calls)
|
||||
}
|
||||
for _, call := range model.calls {
|
||||
if call.thinking {
|
||||
t.Errorf("call %q ran with thinking on; the default must be off", call.prompt)
|
||||
}
|
||||
}
|
||||
// The switch is recorded in the trace too, so a reader of a run can tell
|
||||
// which mode produced a chunk.
|
||||
if got := r.out[imageFieldKeyCaptionThinking]; got != false {
|
||||
t.Errorf(`out[%q] = %v, want false`, imageFieldKeyCaptionThinking, got)
|
||||
}
|
||||
if got := r.out[imageFieldKeyOCRThinking]; got != false {
|
||||
t.Errorf(`out[%q] = %v, want false`, imageFieldKeyOCRThinking, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestThinkingFollowsTheStoredSwitch is the escape hatch: a knowledge base whose
|
||||
// images came out empty can ask for the slower, thinking call without touching
|
||||
// the model, and the switch is honoured for exactly the action that asked.
|
||||
func TestThinkingFollowsTheStoredSwitch(t *testing.T) {
|
||||
r, model := runThinkingPipeline(t, false, map[string]any{
|
||||
captionFieldKeyEnableCaption: true,
|
||||
captionFieldKeyEnableOCR: true,
|
||||
imageFieldKeyCaptionThinking: true,
|
||||
})
|
||||
if len(model.calls) != 2 {
|
||||
t.Fatalf("VLM calls = %d (%v), want 2", len(model.calls), model.calls)
|
||||
}
|
||||
if !model.calls[0].thinking {
|
||||
t.Errorf("caption ran with thinking off; caption_thinking=true did not reach the model")
|
||||
}
|
||||
if model.calls[1].thinking {
|
||||
t.Errorf("OCR ran with thinking on; the switch is per action, and only caption asked")
|
||||
}
|
||||
if got := r.out[imageFieldKeyCaptionThinking]; got != true {
|
||||
t.Errorf(`out[%q] = %v, want true`, imageFieldKeyCaptionThinking, got)
|
||||
}
|
||||
if got := r.out[imageFieldKeyOCRThinking]; got != false {
|
||||
t.Errorf(`out[%q] = %v, want false`, imageFieldKeyOCRThinking, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationPipelineThinkingKeys pins that the thinking switches are
|
||||
// ACTION-level keys shared with the default pipeline: caption.thinking /
|
||||
// ocr.thinking drive the calls, the pre-unification private keys
|
||||
// (describe_thinking / text_thinking) are stale and ignored, and the trace
|
||||
// records the same action keys.
|
||||
func TestObservationPipelineThinkingKeys(t *testing.T) {
|
||||
r, model := runThinkingPipeline(t, true, map[string]any{
|
||||
smartFieldKeyCaptureCaption: true,
|
||||
smartFieldKeyAllowOCR: true,
|
||||
// Stale keys from before the action-level unification: present in a
|
||||
// stored config, honoured by nothing.
|
||||
"describe_thinking": true,
|
||||
"text_thinking": true,
|
||||
// The action-level switches this pipeline declares.
|
||||
imageFieldKeyCaptionThinking: false,
|
||||
imageFieldKeyOCRThinking: true,
|
||||
})
|
||||
if len(model.calls) != 2 {
|
||||
t.Fatalf("VLM calls = %d (%v), want 2", len(model.calls), model.calls)
|
||||
}
|
||||
if model.calls[0].thinking {
|
||||
t.Error("the observation call ran with thinking on; caption.thinking=false was ignored")
|
||||
}
|
||||
if !model.calls[1].thinking {
|
||||
t.Error("the OCR call ran with thinking off; ocr.thinking=true was ignored")
|
||||
}
|
||||
if got := r.out[imageFieldKeyCaptionThinking]; got != false {
|
||||
t.Errorf(`out[%q] = %v, want false`, imageFieldKeyCaptionThinking, got)
|
||||
}
|
||||
if got := r.out[imageFieldKeyOCRThinking]; got != true {
|
||||
t.Errorf(`out[%q] = %v, want true`, imageFieldKeyOCRThinking, got)
|
||||
}
|
||||
// The stale private keys must not leak into the trace.
|
||||
if _, ok := r.out["describe_thinking"]; ok {
|
||||
t.Error("the stale describe_thinking key must not appear in the trace")
|
||||
}
|
||||
if _, ok := r.out["text_thinking"]; ok {
|
||||
t.Error("the stale text_thinking key must not appear in the trace")
|
||||
}
|
||||
}
|
||||
|
|
@ -4443,21 +4443,21 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
|
|||
}
|
||||
|
||||
payload := types.ImageMultimodalPayload{
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kb.ID,
|
||||
ChunkID: chunkID,
|
||||
ImageURL: img.ServingURL,
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
ImageAttrsEnabled: eff.ImageAttrsEnabled,
|
||||
ImageVectorEnabled: &eff.ImageVectorEnabled,
|
||||
ImageActions: eff.ImageActions,
|
||||
Language: lang,
|
||||
ImageSourceType: metadata["image_source_type"],
|
||||
Attempt: attempt,
|
||||
ImageIndex: idx,
|
||||
SourceLocators: img.SourceLocators,
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kb.ID,
|
||||
ChunkID: chunkID,
|
||||
ImageURL: img.ServingURL,
|
||||
ImagePipelineID: eff.ImagePipelineID,
|
||||
ImagePipelineParams: eff.ImagePipelineParams,
|
||||
ImageAttrsEnabled: eff.ImageAttrsEnabled,
|
||||
ImageActions: eff.ImageActions,
|
||||
Language: lang,
|
||||
ImageSourceType: metadata["image_source_type"],
|
||||
Attempt: attempt,
|
||||
ImageIndex: idx,
|
||||
SourceLocators: img.SourceLocators,
|
||||
ImageVectorEnabled: &eff.ImageVectorEnabled,
|
||||
}
|
||||
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro
|
|||
ImageAttrsEnabled: imageCfg.ImageAttrsEnabled,
|
||||
ImageVectorEnabled: imageCfg.ImageVectorEnabled,
|
||||
ImageActions: types.ResolveImageActions(imageCfg.ImageActions),
|
||||
ImagePipelineID: types.ResolveImagePipelineID(&imageCfg),
|
||||
ImagePipelineParams: imageCfg.ImagePipelineParams,
|
||||
}
|
||||
if overrides == nil {
|
||||
return eff
|
||||
|
|
@ -93,6 +95,25 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro
|
|||
}
|
||||
if overrides.ImageAttrsEnabled != nil {
|
||||
eff.ImageAttrsEnabled = *overrides.ImageAttrsEnabled
|
||||
if overrides.ImagePipelineID == nil {
|
||||
// Legacy document overrides still select a pipeline. Parameters
|
||||
// belong to the selected pipeline and cannot follow a switch.
|
||||
id := types.ImagePipelineIDFor(eff.ImageAttrsEnabled)
|
||||
if id != eff.ImagePipelineID {
|
||||
eff.ImagePipelineParams = nil
|
||||
}
|
||||
eff.ImagePipelineID = id
|
||||
}
|
||||
}
|
||||
if overrides.ImagePipelineID != nil {
|
||||
// The pick and its params travel as one unit: params keyed for one
|
||||
// pipeline say nothing to another, so an id without its params would
|
||||
// silently run the new pipeline on the old pipeline's switches.
|
||||
eff.ImagePipelineID = types.NormalizeImagePipelineID(*overrides.ImagePipelineID)
|
||||
eff.ImagePipelineParams = overrides.ImagePipelineParams
|
||||
// The explicit pick also decides the legacy switch, so the fallback
|
||||
// path and the run agree on which pipeline this document asked for.
|
||||
eff.ImageAttrsEnabled = eff.ImagePipelineID == types.ImagePipelineSmartOCR
|
||||
}
|
||||
if overrides.ImageVectorEnabled != nil {
|
||||
eff.ImageVectorEnabled = *overrides.ImageVectorEnabled
|
||||
|
|
@ -161,7 +182,7 @@ func resolveFileImportProcessConfig(
|
|||
}
|
||||
|
||||
if processOverrides != nil {
|
||||
if err := ValidateProcessOverrides(ctx, kb, processOverrides, []string{fileType}); err != nil {
|
||||
if err := validateEffectiveProcessOverrides(eff, []string{fileType}); err != nil {
|
||||
return eff, err
|
||||
}
|
||||
} else if err := validateDefaultFileImportRequirements(ctx, kb, eff, fileType); err != nil {
|
||||
|
|
@ -181,7 +202,12 @@ func ValidateProcessOverrides(
|
|||
if overrides == nil {
|
||||
return nil
|
||||
}
|
||||
return validateEffectiveProcessOverrides(ResolveProcessConfig(kb, overrides), fileTypes)
|
||||
}
|
||||
|
||||
// Validate the final config, including the legacy enableMultimodel argument
|
||||
// applied by upload and reparse callers, before persisting or enqueueing it.
|
||||
func validateEffectiveProcessOverrides(eff types.EffectiveProcessConfig, fileTypes []string) error {
|
||||
hasImage := false
|
||||
hasAudio := false
|
||||
for _, ft := range fileTypes {
|
||||
|
|
@ -193,8 +219,6 @@ func ValidateProcessOverrides(
|
|||
}
|
||||
}
|
||||
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
|
||||
if hasImage {
|
||||
if !eff.VLMConfig.IsEnabled() {
|
||||
return werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
|
||||
|
|
@ -208,6 +232,11 @@ func ValidateProcessOverrides(
|
|||
if err := types.ValidateEffectiveProcessPromptInstructions(eff); err != nil {
|
||||
return werrors.NewBadRequestError(err.Error())
|
||||
}
|
||||
if eff.EnableMultimodel {
|
||||
if err := ValidateImagePipelineParams(eff.ImagePipelineID, eff.ImagePipelineParams); err != nil {
|
||||
return werrors.NewBadRequestError(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -229,7 +258,7 @@ func ApplyKnowledgeProcessOverrides(
|
|||
if processOverrides == nil {
|
||||
return eff, nil
|
||||
}
|
||||
if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil {
|
||||
if err := validateEffectiveProcessOverrides(eff, fileTypes); err != nil {
|
||||
return eff, err
|
||||
}
|
||||
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
|
||||
|
|
|
|||
|
|
@ -161,6 +161,13 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
|
|||
}
|
||||
}
|
||||
|
||||
// The image settings have to survive the same check the settings panel
|
||||
// makes: a pipeline whose every action switch is off would process images
|
||||
// by ignoring them.
|
||||
if err := s.validateImagePipelineConfig(ctx, kb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Creating knowledge base, ID: %s, tenant ID: %d, name: %s", kb.ID, kb.TenantID, kb.Name)
|
||||
|
||||
if err := s.repo.CreateKnowledgeBase(ctx, kb); err != nil {
|
||||
|
|
@ -604,6 +611,11 @@ func (s *knowledgeBaseService) UpdateKnowledgeBase(ctx context.Context,
|
|||
// know about the image settings cannot wipe them.
|
||||
if config.ImageProcessingConfig != nil {
|
||||
kb.ImageProcessingConfig = *config.ImageProcessingConfig
|
||||
// Same rule as on create: the update lands only if the pipeline the
|
||||
// request names can actually run what it declares.
|
||||
if err := s.validateImagePipelineConfig(ctx, kb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if config.FAQConfig != nil {
|
||||
kb.FAQConfig = config.FAQConfig
|
||||
|
|
|
|||
|
|
@ -31,6 +31,14 @@ func (f *fakeVLM) Predict(context.Context, [][]byte, string) (string, error) {
|
|||
return f.response, nil
|
||||
}
|
||||
|
||||
// PredictWithOptions forwards to Predict; these tests pin the cascade between
|
||||
// OCR and caption, not the thinking switch.
|
||||
func (f *fakeVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, _ string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return f.Predict(context.Background(), nil, "")
|
||||
}
|
||||
|
||||
func (f *fakeVLM) GetModelName() string { return "fake-vlm" }
|
||||
func (f *fakeVLM) GetModelID() string { return "fake" }
|
||||
|
||||
|
|
@ -58,6 +66,12 @@ func (f *promptAwareVLM) Predict(_ context.Context, _ [][]byte, prompt string) (
|
|||
return f.ocrResponse, nil
|
||||
}
|
||||
|
||||
func (f *promptAwareVLM) PredictWithOptions(
|
||||
_ context.Context, _ [][]byte, prompt string, _ *vlm.PredictOptions,
|
||||
) (string, error) {
|
||||
return f.Predict(context.Background(), nil, prompt)
|
||||
}
|
||||
|
||||
func (f *promptAwareVLM) GetModelName() string { return "prompt-aware-vlm" }
|
||||
func (f *promptAwareVLM) GetModelID() string { return "fake" }
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/application/access"
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
|
|
@ -1798,3 +1799,67 @@ func (h *KnowledgeBaseHandler) GetImageAttrsSchema(c *gin.Context) {
|
|||
},
|
||||
})
|
||||
}
|
||||
|
||||
// GetImagePipelines returns every registered image pipeline in the shape the
|
||||
// settings panel renders: id, name, and the fields each one owns. The panel
|
||||
// reads this instead of carrying a list of its own, so adding a pipeline is a
|
||||
// backend-only change — its fields arrive here already described, and no UI code
|
||||
// mentions the pipeline by name.
|
||||
//
|
||||
// The fields of two pipelines are deliberately allowed to share a key. Each is
|
||||
// private to the pipeline that declared it, which is why the knowledge base
|
||||
// stores them under that pipeline's own group rather than in one flat map.
|
||||
// Read-only, and global rather than per-KB, so only the Viewer role is required.
|
||||
func (h *KnowledgeBaseHandler) GetImagePipelines(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": service.ListImagePipelines(),
|
||||
})
|
||||
}
|
||||
|
||||
// ValidateImagePipelines answers the settings panel's question — could what is
|
||||
// on screen be saved? — without saving anything. The panel asks as the user
|
||||
// types so the warnings arrive before the save rather than as a rejection after
|
||||
// it; the save path asks again on its own, so this endpoint is the polite half
|
||||
// of one check, not the whole of it.
|
||||
//
|
||||
// Nothing is judged here. The rules come from the pipeline being validated, the
|
||||
// same ones ListImagePipelines publishes, so a pipeline whose actions are not
|
||||
// the user's to switch off declares none and is never told that it is wrong.
|
||||
// Every broken rule is reported, each with the i18n key of its own wording and
|
||||
// the name of the field that is at fault: the panel translates and points, as
|
||||
// it does for the strings already on it.
|
||||
func (h *KnowledgeBaseHandler) ValidateImagePipelines(c *gin.Context) {
|
||||
var request struct {
|
||||
PipelineID types.ImagePipelineID `json:"pipeline_id"`
|
||||
Params map[string]any `json:"params"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"success": false,
|
||||
"message": "invalid request body",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
broken := service.BrokenImagePipelineRules(request.PipelineID, request.Params)
|
||||
if len(broken) == 0 {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": gin.H{"valid": true, "violations": []any{}},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
violations := make([]gin.H, 0, len(broken))
|
||||
for _, invalid := range broken {
|
||||
violations = append(violations, gin.H{
|
||||
"message_key": invalid.MessageKey,
|
||||
"field": invalid.Field,
|
||||
})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": gin.H{"valid": false, "violations": violations},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,9 +22,18 @@ func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
|
|||
func (w *concurrencyVLM) GetModelID() string { return w.inner.GetModelID() }
|
||||
|
||||
func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
return w.PredictWithOptions(ctx, imgBytes, prompt, nil)
|
||||
}
|
||||
|
||||
// PredictWithOptions holds the same concurrency slot around the per-call
|
||||
// variant, so a tuned request (e.g. an explicit thinking switch) is governed
|
||||
// exactly like a plain one.
|
||||
func (w *concurrencyVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytes [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
|
||||
defer release()
|
||||
return w.inner.Predict(ctx, imgBytes, prompt)
|
||||
return w.inner.PredictWithOptions(ctx, imgBytes, prompt, opts)
|
||||
}
|
||||
|
||||
// wrapVLMConcurrency installs the background concurrency governor as the
|
||||
|
|
|
|||
|
|
@ -19,9 +19,15 @@ func (l *langfuseVLM) GetModelName() string { return l.inner.GetModelName() }
|
|||
func (l *langfuseVLM) GetModelID() string { return l.inner.GetModelID() }
|
||||
|
||||
func (l *langfuseVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
return l.PredictWithOptions(ctx, imgBytes, prompt, nil)
|
||||
}
|
||||
|
||||
func (l *langfuseVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytes [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
mgr := langfuse.GetManager()
|
||||
if !mgr.Enabled() {
|
||||
return l.inner.Predict(ctx, imgBytes, prompt)
|
||||
return l.inner.PredictWithOptions(ctx, imgBytes, prompt, opts)
|
||||
}
|
||||
|
||||
totalImgSize := 0
|
||||
|
|
@ -43,7 +49,7 @@ func (l *langfuseVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt str
|
|||
},
|
||||
})
|
||||
|
||||
result, err := l.inner.Predict(genCtx, imgBytes, prompt)
|
||||
result, err := l.inner.PredictWithOptions(genCtx, imgBytes, prompt, opts)
|
||||
|
||||
// VLMs don't return token usage; approximate prompt tokens for cost
|
||||
// tracking in Langfuse (users can configure per-model pricing in the UI).
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
package vlm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type tracingTestVLM struct {
|
||||
spanID trace.SpanID
|
||||
opts *PredictOptions
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *tracingTestVLM) GetModelName() string { return "test-model" }
|
||||
func (m *tracingTestVLM) GetModelID() string { return "test-id" }
|
||||
func (m *tracingTestVLM) Predict(ctx context.Context, images [][]byte, prompt string) (string, error) {
|
||||
return m.PredictWithOptions(ctx, images, prompt, nil)
|
||||
}
|
||||
|
||||
func (m *tracingTestVLM) PredictWithOptions(
|
||||
ctx context.Context, _ [][]byte, _ string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
m.spanID = trace.SpanContextFromContext(ctx).SpanID()
|
||||
m.opts = opts
|
||||
return "caption text", m.err
|
||||
}
|
||||
|
||||
func TestVLMLangfuseRecordsBothPredictionMethods(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
manager, err := langfuse.Init(langfuse.Config{
|
||||
Enabled: true, Host: server.URL, PublicKey: "test", SecretKey: "test",
|
||||
SampleRate: 1, FlushAt: 10, QueueSize: 100, FlushInterval: time.Hour, RequestTimeout: time.Second,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
require.NoError(t, manager.Shutdown(context.Background()))
|
||||
_, err := langfuse.Init(langfuse.Config{})
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
_, parent := manager.Tracer().Start(context.Background(), "fixture")
|
||||
provider := parent.TracerProvider().(*sdktrace.TracerProvider)
|
||||
recorder := tracetest.NewSpanRecorder()
|
||||
provider.RegisterSpanProcessor(recorder)
|
||||
parent.End()
|
||||
|
||||
on := true
|
||||
for _, opts := range []*PredictOptions{nil, {Thinking: &on}} {
|
||||
for _, failure := range []error{nil, errors.New("model unavailable")} {
|
||||
inner := &tracingTestVLM{err: failure}
|
||||
wrapped, err := wrapVLMLangfuse(inner, nil)
|
||||
require.NoError(t, err)
|
||||
wrapped, err = wrapVLMConcurrency(wrapped, 0, nil)
|
||||
require.NoError(t, err)
|
||||
var result string
|
||||
if opts == nil {
|
||||
result, err = wrapped.Predict(context.Background(), nil, "describe this image")
|
||||
} else {
|
||||
result, err = wrapped.PredictWithOptions(context.Background(), nil, "describe this image", opts)
|
||||
}
|
||||
require.Equal(t, failure, err)
|
||||
require.Equal(t, "caption text", result)
|
||||
require.Same(t, opts, inner.opts)
|
||||
require.True(t, inner.spanID.IsValid(), "the model call must receive the generation context")
|
||||
var generation sdktrace.ReadOnlySpan
|
||||
for _, span := range recorder.Ended() {
|
||||
if span.SpanContext().SpanID() == inner.spanID {
|
||||
generation = span
|
||||
}
|
||||
}
|
||||
require.NotNil(t, generation, "the generation must end on both success and error")
|
||||
require.Equal(t, "vlm.predict", generation.Name())
|
||||
attrs := map[string]string{}
|
||||
for _, attr := range generation.Attributes() {
|
||||
attrs[string(attr.Key)] = attr.Value.AsString()
|
||||
}
|
||||
require.Equal(t, "generation", attrs["langfuse.observation.type"])
|
||||
require.Contains(t, attrs["langfuse.observation.usage_details"], "input")
|
||||
if failure != nil {
|
||||
require.Equal(t, codes.Error, generation.Status().Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,8 +15,20 @@ type debugVLM struct {
|
|||
}
|
||||
|
||||
func (d *debugVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
|
||||
return d.call(ctx, imgBytes, prompt, nil)
|
||||
}
|
||||
|
||||
func (d *debugVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytes [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
return d.call(ctx, imgBytes, prompt, opts)
|
||||
}
|
||||
|
||||
func (d *debugVLM) call(
|
||||
ctx context.Context, imgBytes [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
start := time.Now()
|
||||
result, err := d.inner.Predict(ctx, imgBytes, prompt)
|
||||
result, err := d.inner.PredictWithOptions(ctx, imgBytes, prompt, opts)
|
||||
logVLMDebug(ctx, d.inner.GetModelName(), imgBytes, prompt, result, err, time.Since(start))
|
||||
return result, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,14 @@ func NewOllamaVLM(config *Config, ollamaService *ollama.OllamaService) (*OllamaV
|
|||
|
||||
// Predict sends an image with a text prompt to the Ollama vision model.
|
||||
func (v *OllamaVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
|
||||
return v.PredictWithOptions(ctx, imgBytesList, prompt, nil)
|
||||
}
|
||||
|
||||
// PredictWithOptions carries the thinking preference in Ollama's top-level
|
||||
// Think field; sampling parameters remain in Options.
|
||||
func (v *OllamaVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytesList [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
streamFlag := false
|
||||
var images []ollamaapi.ImageData
|
||||
for _, imgBytes := range imgBytesList {
|
||||
|
|
@ -50,6 +58,9 @@ func (v *OllamaVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt s
|
|||
Stream: &streamFlag,
|
||||
Options: map[string]interface{}{"temperature": 0.1},
|
||||
}
|
||||
if opts != nil && opts.Thinking != nil {
|
||||
chatReq.Think = &ollamaapi.ThinkValue{Value: *opts.Thinking}
|
||||
}
|
||||
|
||||
totalImageSize := 0
|
||||
for _, img := range imgBytesList {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
package vlm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOllamaPredictThinkingRequest(t *testing.T) {
|
||||
on, off := true, false
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
opts *PredictOptions
|
||||
want *bool
|
||||
}{
|
||||
{"legacy", nil, nil},
|
||||
{"unset", &PredictOptions{}, nil},
|
||||
{"enabled", &PredictOptions{Thinking: &on}, &on},
|
||||
{"disabled", &PredictOptions{Thinking: &off}, &off},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
requests := make(chan map[string]any, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
requests <- body
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"message":{"role":"assistant","content":"caption"},"done":true}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("OLLAMA_BASE_URL", server.URL)
|
||||
service, err := ollama.GetOllamaService()
|
||||
require.NoError(t, err)
|
||||
model, err := NewOllamaVLM(&Config{ModelName: "test"}, service)
|
||||
require.NoError(t, err)
|
||||
var result string
|
||||
if tc.opts == nil {
|
||||
result, err = model.Predict(context.Background(), nil, "describe")
|
||||
} else {
|
||||
result, err = model.PredictWithOptions(context.Background(), nil, "describe", tc.opts)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "caption", result)
|
||||
body := <-requests
|
||||
if tc.want == nil {
|
||||
require.NotContains(t, body, "think")
|
||||
} else {
|
||||
require.Equal(t, *tc.want, body["think"])
|
||||
}
|
||||
options := body["options"].(map[string]any)
|
||||
require.NotContains(t, options, "think")
|
||||
require.Equal(t, 0.1, options["temperature"])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -93,6 +93,17 @@ func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
|
|||
|
||||
// Predict sends images with a text prompt through the chat client.
|
||||
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
|
||||
return v.PredictWithOptions(ctx, imgBytesList, prompt, nil)
|
||||
}
|
||||
|
||||
// PredictWithOptions sends the same request, carrying a per-call thinking
|
||||
// switch into the chat client. The switch rides on Options.Thinking, which the
|
||||
// protocol layer turns into whatever key this vendor uses, so a model that only
|
||||
// understands chat_template_kwargs still gets that spelling rather than a
|
||||
// bare enable_thinking the server would ignore.
|
||||
func (v *RemoteAPIVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytesList [][]byte, prompt string, opts *PredictOptions,
|
||||
) (string, error) {
|
||||
parts := []chat.MessageContentPart{{Type: "text", Text: prompt}}
|
||||
totalImageSize := 0
|
||||
for _, imgBytes := range imgBytesList {
|
||||
|
|
@ -109,12 +120,20 @@ func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, promp
|
|||
logger.Infof(ctx, "[VLM] Calling chat protocol, model=%s, numImages=%d, totalImageSize=%d",
|
||||
v.modelName, len(imgBytesList), totalImageSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, vlmHTTPTimeout())
|
||||
defer cancel()
|
||||
resp, err := v.chat.Chat(ctx, []chat.Message{{Role: "user", MultiContent: parts}}, &chat.ChatOptions{
|
||||
chatOpts := &chat.ChatOptions{
|
||||
Temperature: v.temperature,
|
||||
MaxTokens: defaultMaxToks,
|
||||
})
|
||||
}
|
||||
// Only an explicit preference is forwarded. The protocol layer skips the
|
||||
// thinking fields entirely when nobody asked for a level, which is what
|
||||
// leaves an always-on reasoning model running as its vendor intends.
|
||||
if opts != nil && opts.Thinking != nil {
|
||||
chatOpts.Thinking = opts.Thinking
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, vlmHTTPTimeout())
|
||||
defer cancel()
|
||||
resp, err := v.chat.Chat(ctx, []chat.Message{{Role: "user", MultiContent: parts}}, chatOpts)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("VLM request: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,18 +13,38 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ErrTruncatedCompletion means the model exhausted its output budget.
|
||||
// ErrTruncatedCompletion means the model exhausted its output budget
|
||||
// (finish_reason=length). Callers can errors.Is it to tell a truncated answer
|
||||
// apart from a genuinely empty one.
|
||||
var ErrTruncatedCompletion = errors.New("VLM completion truncated")
|
||||
|
||||
// VLM defines the interface for Vision Language Model operations.
|
||||
type VLM interface {
|
||||
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
|
||||
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
|
||||
// PredictWithOptions sends the same request with per-call tunings that the
|
||||
// client-level config cannot express. Implementations must honour it; the
|
||||
// decorators below pass it through untouched.
|
||||
PredictWithOptions(ctx context.Context, imgBytes [][]byte, prompt string, opts *PredictOptions) (string, error)
|
||||
|
||||
GetModelName() string
|
||||
GetModelID() string
|
||||
}
|
||||
|
||||
// PredictOptions tunes a single image request. Every field is optional and nil
|
||||
// means "whatever this client was built with", which is what keeps the model
|
||||
// row, the knowledge base and the per-action switch from overriding each other
|
||||
// by accident: an unset field never writes anything.
|
||||
type PredictOptions struct {
|
||||
// Thinking pins the thinking switch for this call. Leaving a reasoning model
|
||||
// on costs a vision call more than it returns it: the completion budget goes
|
||||
// to reasoning, and at the vision default temperature the answer can come
|
||||
// back empty after the model has already listed the answer in its reasoning.
|
||||
// An OCR caller cannot tell that apart from an image with no text, so the
|
||||
// switch is per-action rather than per-model.
|
||||
Thinking *bool
|
||||
}
|
||||
|
||||
// Config holds the configuration needed to create a VLM instance.
|
||||
type Config struct {
|
||||
Source types.ModelSource
|
||||
|
|
|
|||
|
|
@ -92,6 +92,15 @@ type weKnoraCloudVLMResponse struct {
|
|||
|
||||
// Predict sends images with a text prompt to the WeKnoraCloud API.
|
||||
func (v *WeKnoraCloudVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
|
||||
return v.PredictWithOptions(ctx, imgBytesList, prompt, nil)
|
||||
}
|
||||
|
||||
// PredictWithOptions satisfies the interface. The hosted endpoint exposes no
|
||||
// thinking switch, so the options are accepted and used for nothing — the same
|
||||
// call goes out as Predict sends.
|
||||
func (v *WeKnoraCloudVLM) PredictWithOptions(
|
||||
ctx context.Context, imgBytesList [][]byte, prompt string, _ *PredictOptions,
|
||||
) (string, error) {
|
||||
var parts []weKnoraCloudVLMContentPart
|
||||
|
||||
parts = append(parts, weKnoraCloudVLMContentPart{
|
||||
|
|
|
|||
|
|
@ -281,6 +281,18 @@ func RegisterKnowledgeBaseRoutes(r *gin.RouterGroup, handler *handler.KnowledgeB
|
|||
func RegisterImageAttrRoutes(r *gin.RouterGroup, handler *handler.KnowledgeBaseHandler, g *rbacGuards) {
|
||||
g.apiKeyRoute(r, http.MethodGet, "/image-attrs/schema",
|
||||
apiKeyRetrieve(apiKeyFullAccess()), g.Viewer(), handler.GetImageAttrsSchema)
|
||||
// Same shape and the same audience as the schema above: a read-only list of
|
||||
// what the settings panel can offer, with no KB id, so the Viewer role is
|
||||
// enough. Mounted here rather than under /knowledge-bases because neither
|
||||
// the list nor a pipeline id is knowledge-base-specific.
|
||||
g.apiKeyRoute(r, http.MethodGet, "/image-pipelines",
|
||||
apiKeyRetrieve(apiKeyFullAccess()), g.Viewer(), handler.GetImagePipelines)
|
||||
// The settings panel asks this while the user edits, so that a combination
|
||||
// the pipeline cannot run is pointed out before the save rather than
|
||||
// rejected by it. Read-only, global, and answered from the pipeline's own
|
||||
// rules, so only the Viewer role is required.
|
||||
g.apiKeyRoute(r, http.MethodPost, "/image-pipelines/validate",
|
||||
apiKeyRetrieve(apiKeyFullAccess()), g.Viewer(), handler.ValidateImagePipelines)
|
||||
}
|
||||
|
||||
// RegisterKnowledgeBaseActivityRoutes exposes the read-only per-KB activity
|
||||
|
|
|
|||
|
|
@ -1,67 +1,249 @@
|
|||
package types
|
||||
|
||||
// ActionKind enumerates the normalized image-pipeline actions. Under the
|
||||
// unified action framework every image step is one of these; the dispatch loop
|
||||
// treats them uniformly so adding an action is a registry entry, not a new
|
||||
// if/else branch.
|
||||
type ActionKind string
|
||||
// This file is the vocabulary of the image pipeline layer: the ids of the
|
||||
// actions a pipeline can run, the declarative shape of an action, and the ids of
|
||||
// the pipelines themselves. What an action *does* is bound by the action
|
||||
// registry in application/service (image_action_registry.go) — keeping the two
|
||||
// halves apart is what lets this package stay free of any transport dependency.
|
||||
|
||||
// ImageActionID is the dispatch key of an image action. A pipeline references an
|
||||
// action by this id; which code runs for that id is decided by the registry, so
|
||||
// adding an action is one entry here plus a binding, not a new branch in a loop.
|
||||
type ImageActionID string
|
||||
|
||||
const (
|
||||
// ActionCaption asks the model for a short description of the image.
|
||||
ActionCaption ActionKind = "caption"
|
||||
// ActionObservation asks the model to observe the registered attributes
|
||||
// (contain.text, contain.data_visual, ...). It is the only action that
|
||||
// feeds a later decision — the OCR policy reads its result.
|
||||
ActionObservation ActionKind = "observation"
|
||||
// ActionOCR extracts the text the image carries, when the policy wants it.
|
||||
ActionOCR ActionKind = "ocr"
|
||||
// ImageActionCaption asks the model for a short description of the image
|
||||
// and stores it as the caption slot.
|
||||
ImageActionCaption ImageActionID = "caption"
|
||||
// ImageActionObservationCaption observes the registered attributes
|
||||
// (contain.text, contain.data_visual, ...) and describes the image in the
|
||||
// same answer. One VLM request fills the attribute slots and the caption
|
||||
// slot, which is why there is no standalone observation action and no
|
||||
// "merge observation with caption" rule anywhere in the framework.
|
||||
ImageActionObservationCaption ImageActionID = "observation_caption"
|
||||
// ImageActionOCR extracts the text the image carries, when the policy wants
|
||||
// it.
|
||||
ImageActionOCR ImageActionID = "ocr"
|
||||
)
|
||||
|
||||
// PipelineMode labels which image pipeline a trace row ran under. It is
|
||||
// recorded on the per-image subspan input so a reader of the processing trace
|
||||
// can tell an attribute-observed run from the caption+OCR fallback: reading the
|
||||
// stored image_info alone cannot, because a run that observed the attributes
|
||||
// and then skipped OCR looks exactly like one that never observed anything.
|
||||
type PipelineMode string
|
||||
// ImageActionImpl records how an action does its work. The flag is not
|
||||
// decoration: an action that calls a model to check an existing value has
|
||||
// stopped checking and started generating again.
|
||||
type ImageActionImpl string
|
||||
|
||||
const (
|
||||
// PipelineObservationDriven observes the registered attributes first, then
|
||||
// OCRs only when the attribute policy says so.
|
||||
PipelineObservationDriven PipelineMode = "observation_driven"
|
||||
// PipelineCaptionOCR is the upstream path: caption every image, then OCR
|
||||
// every image, with no observation and no policy.
|
||||
PipelineCaptionOCR PipelineMode = "caption_ocr"
|
||||
// ImageActionImplPure runs without a model.
|
||||
ImageActionImplPure ImageActionImpl = "pure"
|
||||
// ImageActionImplVLM spends one or more model requests.
|
||||
ImageActionImplVLM ImageActionImpl = "vlm"
|
||||
)
|
||||
|
||||
// PipelineModeFor maps the attribute-observation switch to its trace label, so
|
||||
// the label and the action table below can never disagree about which pipeline
|
||||
// a task ran under.
|
||||
func PipelineModeFor(attrsEnabled bool) PipelineMode {
|
||||
if attrsEnabled {
|
||||
return PipelineObservationDriven
|
||||
}
|
||||
return PipelineCaptionOCR
|
||||
// ImageActionSpec is the declarative half of an action: what it is allowed to
|
||||
// write, what it declares to read, and the knobs a settings panel may expose.
|
||||
// The executable half is bound by the registry.
|
||||
type ImageActionSpec struct {
|
||||
// ID dispatches the action inside a pipeline; it is also how a pipeline
|
||||
// refers to it and how the gallery and the settings panel name it.
|
||||
ID ImageActionID
|
||||
// Impl says whether the action may call a model.
|
||||
Impl ImageActionImpl
|
||||
// Writes lists the output slots the action fills; nil means it writes none.
|
||||
// Two actions of one pipeline may write the same slot on purpose — the
|
||||
// iterate and validate shapes both overwrite what they just read — so
|
||||
// uniqueness is deliberately not asserted here.
|
||||
Writes []string
|
||||
// Reads lists the slots the action consumes. Declaration only: it produces
|
||||
// trace output, it does not order the pipeline.
|
||||
Reads []string
|
||||
// Version fingerprints the prompts and rules the action encodes, so a trace
|
||||
// row can be matched back to the revision that produced an image rather
|
||||
// than only to the pipeline that ran.
|
||||
Version string
|
||||
// Fields are the per-action tunables the settings panel renders.
|
||||
Fields []ImageFieldDef
|
||||
}
|
||||
|
||||
// DefaultEnabledActions returns the per-action default-on flags for a pipeline
|
||||
// mode. Observation mode observes and captions by default and lets the OCR
|
||||
// decision gate OCR; caption+OCR mode captions and OCRs every image. This is
|
||||
// the single source of default action membership — a new action is enabled
|
||||
// here, never scattered across call sites.
|
||||
// ImageFieldDef is one tunable of an action as the settings panel sees it. Its
|
||||
// json tags are snake_case to match the rest of the image API, including the
|
||||
// enclosing ImagePipelineSpec.
|
||||
type ImageFieldDef struct {
|
||||
// Key is the name the action reads with r.Param(key).
|
||||
Key string `json:"key"`
|
||||
// Type selects the control the panel renders.
|
||||
Type ImageFieldType `json:"type"`
|
||||
// Label and Description are written in the project's default language; the
|
||||
// frontend overlays its translations and falls back to these.
|
||||
Label string `json:"label"`
|
||||
Description string `json:"description"`
|
||||
// Default applies when the knowledge base does not override the field.
|
||||
Default any `json:"default"`
|
||||
// Options enumerates the allowed values of an enum field.
|
||||
Options []string `json:"options"`
|
||||
// DecidesAction marks a field that turns one of the pipeline's actions on or
|
||||
// off, as opposed to one that tunes a call the pipeline always makes. Only
|
||||
// such a field can leave a pipeline with nothing left to do, and only such
|
||||
// a field is named by the rules of ImagePipelineRules.
|
||||
DecidesAction bool `json:"decides_action"`
|
||||
}
|
||||
|
||||
// ImageFieldType is the control a tunable renders as.
|
||||
type ImageFieldType string
|
||||
|
||||
const (
|
||||
// ImageFieldTypeBool renders as a switch or checkbox.
|
||||
ImageFieldTypeBool ImageFieldType = "bool"
|
||||
// ImageFieldTypeString renders as a text input.
|
||||
ImageFieldTypeString ImageFieldType = "string"
|
||||
// ImageFieldTypeEnum renders as a select over the field's Options.
|
||||
ImageFieldTypeEnum ImageFieldType = "enum"
|
||||
)
|
||||
|
||||
// SharedImageActions are the actions every pipeline starts from. They are
|
||||
// declared here and bound to handlers by the registry; an action that belongs to
|
||||
// a single pipeline instead registers under a "<pipeline_id>.<action>" id.
|
||||
//
|
||||
// Note the deliberate parallel with DefaultImageActions: this is which actions
|
||||
// run at all, that one is the attribute -> work policy that gates OCR.
|
||||
func DefaultEnabledActions(attrsEnabled bool) map[ActionKind]bool {
|
||||
if attrsEnabled {
|
||||
return map[ActionKind]bool{
|
||||
ActionCaption: true,
|
||||
ActionObservation: true,
|
||||
ActionOCR: false,
|
||||
}
|
||||
}
|
||||
return map[ActionKind]bool{
|
||||
ActionCaption: true,
|
||||
ActionObservation: false,
|
||||
ActionOCR: true,
|
||||
}
|
||||
// Version carries the placeholder revision "v1" until the prompts move into the
|
||||
// actions and can be fingerprinted there.
|
||||
var SharedImageActions = []ImageActionSpec{
|
||||
{
|
||||
ID: ImageActionCaption,
|
||||
Impl: ImageActionImplVLM,
|
||||
Writes: []string{"caption"},
|
||||
Version: "v1",
|
||||
},
|
||||
{
|
||||
ID: ImageActionObservationCaption,
|
||||
Impl: ImageActionImplVLM,
|
||||
Writes: []string{"attrs", "caption"},
|
||||
// Observing reads the image and the registered attribute table; the
|
||||
// caption is a by-product of that same request.
|
||||
Reads: []string{"attrs"},
|
||||
Version: "v1",
|
||||
},
|
||||
{
|
||||
ID: ImageActionOCR,
|
||||
Impl: ImageActionImplVLM,
|
||||
Writes: []string{"ocr_text"},
|
||||
// The OCR prompt is deliberately not built from the knowledge base
|
||||
// custom instructions, so it does not read the same slot the other two
|
||||
// does. See buildVLMOCRPrompt.
|
||||
Version: "v1",
|
||||
},
|
||||
}
|
||||
|
||||
// ImagePipelineID labels which image pipeline produced a run. It is recorded on
|
||||
// the per-image subspan input, because reading the stored image_info alone cannot
|
||||
// distinguish an image whose pipeline observed the attributes and then skipped
|
||||
// OCR from one whose pipeline never observed anything.
|
||||
type ImagePipelineID string
|
||||
|
||||
const (
|
||||
// ImagePipelineSmartOCR observes the registered attributes first, captions
|
||||
// in the same answer, then OCRs only when the attribute policy allows it.
|
||||
// The id promises that observation comes before the decision; it does not
|
||||
// promise that those three actions are the whole story.
|
||||
ImagePipelineSmartOCR ImagePipelineID = "smartocr"
|
||||
// ImagePipelineDefault is the manual path: the user picks which parsing
|
||||
// actions run, and each runs for every image with no observation and no
|
||||
// policy. It is the fallback whenever nothing else has been chosen.
|
||||
ImagePipelineDefault ImagePipelineID = "default"
|
||||
// ImagePipelineLegacyObCapOCR is the id ImagePipelineSmartOCR carried
|
||||
// before it was renamed to match the label the panel shows. Configs saved
|
||||
// since 2026-09 may name it, so it is normalized rather than honoured:
|
||||
// see NormalizeImagePipelineID.
|
||||
ImagePipelineLegacyObCapOCR ImagePipelineID = "ob_cap_ocr"
|
||||
// ImagePipelineLegacyCaptionOCR is the id ImagePipelineDefault carried
|
||||
// before the panel was reworked. Stored configs may still name it, so it
|
||||
// is normalized rather than honoured: see NormalizeImagePipelineID.
|
||||
ImagePipelineLegacyCaptionOCR ImagePipelineID = "caption_ocr"
|
||||
)
|
||||
|
||||
// NormalizeImagePipelineID maps a stored pipeline id onto the id this build
|
||||
// registers. A rename must not strand the knowledge bases that stored the old
|
||||
// spelling, so every id that once named a shipped pipeline keeps resolving to
|
||||
// its successor; unknown ids pass through unchanged and the caller falls back.
|
||||
func NormalizeImagePipelineID(id ImagePipelineID) ImagePipelineID {
|
||||
switch id {
|
||||
case ImagePipelineLegacyObCapOCR:
|
||||
return ImagePipelineSmartOCR
|
||||
case ImagePipelineLegacyCaptionOCR:
|
||||
return ImagePipelineDefault
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// ImagePipelineSpec is how a pipeline presents itself to the settings panel.
|
||||
// The frontend reads this shape instead of knowing any pipeline by name, so
|
||||
// adding a pipeline is one registered file plus its translation, with no UI
|
||||
// change; Fields carry the same shapes the panel renders today.
|
||||
type ImagePipelineSpec struct {
|
||||
// ID matches imagePipeline.ID(); it is what the knowledge base stores.
|
||||
ID ImagePipelineID `json:"id"`
|
||||
// Name is the human-readable label; the frontend overlays its translation
|
||||
// and falls back to this text.
|
||||
Name string `json:"name"`
|
||||
// Description tells the user how this pipeline works, shown under the
|
||||
// pick. Like Name it is overlay-translated with this text as fallback.
|
||||
Description string `json:"description"`
|
||||
// Fields are this pipeline's private tunables. The frontend renders one
|
||||
// control per field and posts them back under the pipeline's own key, so
|
||||
// two pipelines can each own a field named "enable_ocr" without either
|
||||
// seeing the other's.
|
||||
Fields []ImageFieldDef `json:"fields"`
|
||||
// Rules are conditions over those fields that have to hold before the
|
||||
// knowledge base can save them. Empty means the pipeline accepts any
|
||||
// combination of what it offers.
|
||||
Rules []ImagePipelineRules `json:"rules,omitempty"`
|
||||
}
|
||||
|
||||
// ImagePipelineIDFor maps the attribute-observation switch to a pipeline id, so
|
||||
// the trace label and the pipeline that actually runs can never disagree.
|
||||
func ImagePipelineIDFor(attrsEnabled bool) ImagePipelineID {
|
||||
if attrsEnabled {
|
||||
return ImagePipelineSmartOCR
|
||||
}
|
||||
return ImagePipelineDefault
|
||||
}
|
||||
|
||||
// ImagePipelineRules are the conditions a pipeline's tunables have to meet
|
||||
// before they can be saved. They travel with the pipeline's spec, and the
|
||||
// settings panel evaluates them generically: the frontend knows no rule of its
|
||||
// own, it only reads what the pipeline here declared and reports the result.
|
||||
// A pipeline whose choices cannot conflict — the smart one schedules its own
|
||||
// actions — declares none, and no switch it offers can ever be called invalid.
|
||||
type ImagePipelineRules struct {
|
||||
// AtLeastOne is satisfied when at least one field of Fields is on, and
|
||||
// fails when all of them are off: a set of switches that would leave the
|
||||
// image with nothing done to it.
|
||||
AtLeastOne []string `json:"at_least_one,omitempty"`
|
||||
// MessageKey is the frontend i18n key for the warning shown when this rule
|
||||
// is not met.
|
||||
MessageKey string `json:"message_key"`
|
||||
// Field names the control the panel points at, so the user is told which
|
||||
// switch to turn back on rather than simply that something is wrong.
|
||||
// The first of Fields when it is left empty.
|
||||
Field string `json:"field,omitempty"`
|
||||
}
|
||||
|
||||
// ImagePipelineValidationError is what a pipeline returns when the tunables a
|
||||
// knowledge base saved for it cannot be run as they stand. It carries an i18n
|
||||
// key rather than a sentence: the settings panel asks the backend whether what
|
||||
// it shows can be saved, and every string on that panel is translated on the
|
||||
// frontend — a bare English sentence here would be the one string in the UI
|
||||
// that escapes the overlay.
|
||||
type ImagePipelineValidationError struct {
|
||||
// MessageKey is the frontend i18n key naming what is wrong.
|
||||
MessageKey string `json:"message_key"`
|
||||
// Field names the field at fault when a single one is. The panel points at
|
||||
// that control instead of at the section, which is what makes "switch one
|
||||
// of them back on" actionable. Empty means the whole configuration is at
|
||||
// fault.
|
||||
Field string `json:"field,omitempty"`
|
||||
}
|
||||
|
||||
func (e *ImagePipelineValidationError) Error() string {
|
||||
if e.Field == "" {
|
||||
return "image pipeline configuration is invalid: " + e.MessageKey
|
||||
}
|
||||
return "image pipeline field " + e.Field + " is invalid: " + e.MessageKey
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,14 @@ type KnowledgeProcessOverrides struct {
|
|||
// pipeline switch for this one document. nil keeps the knowledge base's
|
||||
// setting.
|
||||
ImageAttrsEnabled *bool `json:"image_attrs_enabled,omitempty"`
|
||||
// ImagePipelineID overrides which image pipeline runs for this one
|
||||
// document. nil keeps the knowledge base's resolved pipeline. When set,
|
||||
// ImagePipelineParams travels with it: the params of one pipeline are
|
||||
// meaningless to another, so the pair is replaced as a unit.
|
||||
ImagePipelineID *ImagePipelineID `json:"image_pipeline,omitempty"`
|
||||
// ImagePipelineParams are the overridden pipeline's private tunables. They
|
||||
// are only applied together with ImagePipelineID.
|
||||
ImagePipelineParams map[string]any `json:"image_pipeline_params,omitempty"`
|
||||
// ImageVectorEnabled overrides image vector indexing for this document.
|
||||
// It does not change the KB-wide retrieval setting or embedding model.
|
||||
ImageVectorEnabled *bool `json:"image_vector_enabled,omitempty"`
|
||||
|
|
@ -41,5 +49,7 @@ type EffectiveProcessConfig struct {
|
|||
ImageAttrsEnabled bool
|
||||
ImageVectorEnabled bool
|
||||
ImageActions ImageActionsConfig
|
||||
ImagePipelineID ImagePipelineID
|
||||
ImagePipelineParams map[string]any
|
||||
ExtractConfig ExtractConfig
|
||||
}
|
||||
|
|
|
|||
|
|
@ -560,6 +560,17 @@ type ImageProcessingConfig struct {
|
|||
// override it per document through
|
||||
// KnowledgeProcessOverrides.ImageAttrsEnabled.
|
||||
ImageAttrsEnabled bool `yaml:"image_attrs_enabled,omitempty" json:"image_attrs_enabled,omitempty" extensions:"x-agent"` //nolint:lll // one-line struct tag
|
||||
// ImagePipelineID selects the image pipeline by id, so the choice the
|
||||
// settings panel offers is the choice the backend runs. It is unset on
|
||||
// knowledge bases configured before the field existed; those follow
|
||||
// ImageAttrsEnabled — see ResolveImagePipelineID. An explicit id always
|
||||
// wins over the switch.
|
||||
ImagePipelineID string `yaml:"image_pipeline,omitempty" json:"image_pipeline,omitempty"`
|
||||
// ImagePipelineParams holds the tunables of ImagePipelineID, keyed by the
|
||||
// field's own key. The map travels whole: a key a pipeline no longer
|
||||
// declares is dropped when the run reads it, so an old stored config stays
|
||||
// loadable rather than being rewritten on save.
|
||||
ImagePipelineParams map[string]any `yaml:"image_pipeline_params,omitempty" json:"image_pipeline_params,omitempty"`
|
||||
// ImageVectorEnabled lets a knowledge base whose embedding model takes
|
||||
// images index each image by its own vector and recall it with one. It
|
||||
// is off by default on purpose: a model that takes images is also used
|
||||
|
|
@ -569,6 +580,22 @@ type ImageProcessingConfig struct {
|
|||
ImageVectorEnabled bool `yaml:"image_vector_enabled,omitempty" json:"image_vector_enabled,omitempty" extensions:"x-agent"` //nolint:lll // one-line struct tag
|
||||
}
|
||||
|
||||
// ResolveImagePipelineID returns the pipeline id a knowledge base asks for. An
|
||||
// explicit ImagePipelineID wins; without one the old attribute-observation
|
||||
// switch still decides, so a knowledge base saved before the selector existed
|
||||
// runs the pipeline the switch pointed at. An id that no pipeline registers is
|
||||
// ignored here and falls back to the switch, which is what keeps a stored
|
||||
// config from pointing at a pipeline that has since been removed.
|
||||
func ResolveImagePipelineID(cfg *ImageProcessingConfig) ImagePipelineID {
|
||||
if cfg == nil {
|
||||
return ImagePipelineIDFor(false)
|
||||
}
|
||||
if id := strings.TrimSpace(cfg.ImagePipelineID); id != "" {
|
||||
return NormalizeImagePipelineID(ImagePipelineID(id))
|
||||
}
|
||||
return ImagePipelineIDFor(cfg.ImageAttrsEnabled)
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert ChunkingConfig to database value
|
||||
func (c ChunkingConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(c)
|
||||
|
|
|
|||
|
|
@ -510,14 +510,24 @@ type ImageMultimodalPayload struct {
|
|||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
ImageURL string `json:"image_url"` // provider:// URL (e.g. local://..., minio://...)
|
||||
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
// EnableOCR and EnableCaption are whole-task switches applied on top of the
|
||||
// image-action policy: a step runs only when both this switch and the
|
||||
// policy allow it. Normal parsing leaves both true and lets the observed
|
||||
// attributes decide.
|
||||
EnableOCR bool `json:"enable_ocr"`
|
||||
EnableCaption bool `json:"enable_caption"`
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
ImageSourceType string `json:"image_source_type,omitempty"` // Source type of the image (e.g., "scanned_pdf")
|
||||
// ImagePipelineParams carries the tunables of the pipeline this task runs,
|
||||
// resolved from the knowledge base at enqueue time. They are deliberately
|
||||
// not payload fields of their own: two pipelines may each define a field
|
||||
// called "enable_ocr" with a different meaning, so a name only has to be
|
||||
// unambiguous relative to the pipeline that reads it. A key the running
|
||||
// pipeline does not declare is ignored, which is what lets a stored
|
||||
// knowledge base keep working after a pipeline stops reading one.
|
||||
// ImagePipelineID is the pipeline this task runs, resolved from the
|
||||
// knowledge base at enqueue time by types.ResolveImagePipelineID. It
|
||||
// travels in the payload rather than being re-read at handle time for the
|
||||
// same reason ImageAttrsEnabled does: a task already in flight was
|
||||
// enqueued for one pipeline and must not be re-routed by a later edit.
|
||||
ImagePipelineID ImagePipelineID `json:"image_pipeline,omitempty"`
|
||||
ImagePipelineParams map[string]any `json:"image_pipeline_params,omitempty"`
|
||||
// Language is the request locale for {{language}} in prompt templates.
|
||||
Language string `json:"language,omitempty"`
|
||||
// ImageSourceType drives the OCR prompt, e.g. "scanned_pdf".
|
||||
ImageSourceType string `json:"image_source_type,omitempty"`
|
||||
// Attempt links this image task back to the parent ProcessDocument
|
||||
// attempt so the worker can record its image[i] subspan under the
|
||||
// same attempt's multimodal stage span.
|
||||
|
|
@ -527,7 +537,7 @@ type ImageMultimodalPayload struct {
|
|||
// ("multimodal.image[3]") so the timeline preserves order.
|
||||
ImageIndex int `json:"image_index,omitempty"`
|
||||
// ImageAttrsEnabled selects which image pipeline the task runs — see
|
||||
// PipelineModeFor and the PipelineMode constants. True is the
|
||||
// types.ImagePipelineIDFor and the ImagePipelineID constants. True is the
|
||||
// attribute-observed pipeline: the describe round also observes image
|
||||
// attributes, and the attribute policy decides whether OCR runs for it.
|
||||
// False is the upstream behaviour: a plain caption, then OCR, no attribute
|
||||
|
|
|
|||
Loading…
Reference in New Issue