diff --git a/.gitignore b/.gitignore index 02d23e33d..58d62a5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -194,5 +194,6 @@ config.yml # Local verification outputs; release sources remain visible to Git. /ui/dist*/ /ui/.summary-build/ +/ui/.codex-validation*/ /jieba.cache /mcp-engine-*/ diff --git a/deploy_artifacts/DEPLOYED_20261110.md b/deploy_artifacts/DEPLOYED_20261110.md new file mode 100644 index 000000000..bc4dc7b19 --- /dev/null +++ b/deploy_artifacts/DEPLOYED_20261110.md @@ -0,0 +1,40 @@ +# 52.118 Docker 升级记录(2026-11-10) + +## 当前版本 + +- MaxKB 源码:`v3.2-refact`,提交 `79fa2cf4a27c0363946317af8c8e5c49f50fee0d`。 +- markitdown 源码:`develop-mineru-2.7.6`,提交 `94d2f22b3c3930e2c9bbcd9aaf1d85b0c1ff9ed5`。 +- 发布版本:`refact-276-20261110-1`;容器 `airag`、`mineru-api`、`unis-hub-worker`、`unis-gateway`。 +- 离线包:`/home/tlw/unis/deploy_artifacts_upgrade/refact-276-20261109-1-cuda/package-refact-276-20261110-1`。 +- 原数据:`/home/tlw/unis/maxkb-fresh-init-20261109`;当前运行的数据副本:`/home/tlw/unis/maxkb-refact-276-20261110-1`。 +- 原模型目录:`/home/tlw/unis/mineru-models`,只读挂载,未覆盖。 +- 私有备份与回退状态:`/home/tlw/unis/releases/refact-276-20261110-1-cuda`;含旧容器配置及敏感环境信息,勿公开。 + +## 打包调整 + +目标分支通过服务器远程仓库克隆到独立目录,未覆盖原源码目录。最终包由 `build-compat-release.sh` 基于已构建的 `refact-276-20261109-1` 镜像生成,`BASE_IMAGES.txt` 记录基础镜像 ID,`SOURCES.txt` 记录源码提交,`SHA256SUMS.txt` 覆盖交付文件。原基础发布包和构建文件仍保留在同一发布目录下的 `package-retry8`。 + +知识库镜像固定 Pydantic 2.12.3,避免 MCP 1.19.0 引用被 Pydantic 2.14 移除的接口。CUDA 镜像使用 Gradio 5.50.0 解决目标分支的 Gradio/Pydantic 版本冲突,并补装基础镜像缺少的 pycairo;构建时检查加速器依赖和 `pip check`。NVIDIA 容器需显式使用 `nvidia` runtime。 + +## 执行与验证 + +旧版 PostgreSQL 已正常关闭并以 `pg_controldata` 验证关闭状态,再制作归档和独立副本。新版本在副本上完成迁移,旧版四个容器已改名保留。`airag` 和 `mineru-api` 健康,网关与 Hub worker 在运行;GPU 检查识别到一块 NVIDIA 设备。 + +服务器侧 8989 的登录、注册、聊天入口返回 200;`/mindmap/` 和 9898 返回工作台重定向。容器内 8080 登录及注册入口返回 200;宿主机 8080 未监听,不能将其描述为已在宿主机开放。浏览器已检查登录和注册页面;admin 首页引用的入口静态资源可访问。 + +通过 `pipeline` 调用 MinerU 2.7.6,使用临时生成的一页英文 PDF,接口返回 200 且识别出测试文字。未验证普通用户真实登录、向量检索、图片 OCR、Word、VLM、总结、保存和导出等端到端流程。平台默认解析服务和大模型仍需按需配置。 + +后续发现迁移只创建了统一角色表,没有执行旧版管理员导入。已在当前容器执行项目内 `unify_platform_roles`,恢复原有 `admin` 的统一管理员身份;实际登录页面已确认 AI 应用、工具、模型和系统管理入口恢复。升级脚本已增加这一初始化步骤;已有管理员角色时不会重新导入,避免撤销的管理员权限被重新授予。 + +按后续要求,已执行可选增量脚本 `seed_local_v2_parsers.py`,登记“传统管道”“混合引擎”“VLM引擎”三种 2.7.6 解析资源,分别授权内置管理员和普通用户角色查看、使用。重复执行未新增记录;三个资源的接口连接和角色权限已检查。Pipeline 已用测试 PDF 验证;Hybrid 和 VLM 的实际模型加载与推理尚未验收。脚本不设置工作台默认解析器。 + +## 回退 + +先停止新版本写入并确认新版数据处理方案。回退后旧版数据不会自动包含升级后新写入的数据: + +```bash +cd /home/tlw/unis/deploy_artifacts_upgrade/refact-276-20261109-1-cuda/package-refact-276-20261110-1 +sudo python3 upgrade.py rollback --release-dir /home/tlw/unis/releases/refact-276-20261110-1-cuda --start-old +``` + +脚本会校验容器身份、停止并保留新版容器及副本,恢复原容器名称并尝试启动原版;若旧数据存在其他使用方会拒绝启动。执行前应复核状态,不要手工删除备份、模型或新旧数据目录。 diff --git a/deploy_artifacts/Dockerfile.kb b/deploy_artifacts/Dockerfile.kb index f3a617714..d69be461b 100644 --- a/deploy_artifacts/Dockerfile.kb +++ b/deploy_artifacts/Dockerfile.kb @@ -1,7 +1,7 @@ FROM node:24-alpine AS web-build WORKDIR /build/ui COPY ui/package.json ui/pnpm-lock.yaml ui/pnpm-workspace.yaml ./ -RUN npm install -g pnpm@11.23.0 && pnpm install --frozen-lockfile +RUN npm install -g pnpm@11.23.0 && pnpm install --frozen-lockfile --shamefully-hoist COPY ui/ ./ # Never reuse dist or Vite's relative development base for a release. RUN rm -rf dist && npm run build-only -- --base /admin/ && npm run build-only-chat -- --base /chat/ && \ @@ -9,11 +9,17 @@ RUN rm -rf dist && npm run build-only -- --base /admin/ && npm run build-only-ch grep -q '/admin/assets/' dist/admin/index.html FROM ghcr.io/1panel-dev/maxkb-base:python3.11-pg17.6 AS backend-build +ARG UV_INDEX_URL +ENV UV_HTTP_TIMEOUT=30 UV_HTTP_RETRIES=2 WORKDIR /opt/maxkb-app COPY . . -RUN apt-get update && apt-get install -y --no-install-recommends gcc g++ gettext libexpat1-dev libffi-dev && \ - gcc -shared -fPIC -o ${MAXKB_SANDBOX_HOME}/sandbox.so installer/sandbox.c -ldl && \ - pip install uv --break-system-packages && python -m uv pip install -r pyproject.toml && \ +RUN printf 'deb https://mirrors.aliyun.com/debian trixie main\n' > /tmp/unis-build.list && \ + timeout 180s apt-get -o Dir::Etc::sourcelist=/tmp/unis-build.list -o Dir::Etc::sourceparts=- -o Acquire::ForceIPv4=true -o Acquire::https::Timeout=30 -o Acquire::Retries=1 update && \ + timeout 180s apt-get -o Dir::Etc::sourcelist=/tmp/unis-build.list -o Dir::Etc::sourceparts=- -o Acquire::ForceIPv4=true -o Acquire::https::Timeout=30 -o Acquire::Retries=1 install -y --no-install-recommends gcc g++ gettext libexpat1-dev libffi-dev +RUN gcc -shared -fPIC -o ${MAXKB_SANDBOX_HOME}/sandbox.so installer/sandbox.c -ldl && \ + pip install uv --break-system-packages && printf 'pydantic==2.12.3\n' > /tmp/unis-kb-constraints.txt && \ + timeout 600s python -m uv pip install -c /tmp/unis-kb-constraints.txt -r pyproject.toml && \ + python -c 'from pydantic._internal._typing_extra import eval_type_backport; import mcp.types' && python -m pip check && \ python apps/manage.py compilemessages && \ python -m uv pip install --target=/opt/maxkb-app/sandbox/python-packages requests pymysql psycopg2-binary && \ rm -rf ui installer deploy_artifacts && rm -rf /var/lib/apt/lists/* diff --git a/deploy_artifacts/Dockerfile.kb.compat b/deploy_artifacts/Dockerfile.kb.compat new file mode 100644 index 000000000..6ae0d5a90 --- /dev/null +++ b/deploy_artifacts/Dockerfile.kb.compat @@ -0,0 +1,7 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +ARG UV_INDEX_URL +ENV UV_HTTP_TIMEOUT=30 UV_HTTP_RETRIES=2 +RUN timeout 300s python -m uv pip install 'pydantic==2.12.3' && \ + python -c 'from pydantic._internal._typing_extra import eval_type_backport; import mcp.types' && \ + python -m pip check diff --git a/deploy_artifacts/Dockerfile.kb.dockerignore b/deploy_artifacts/Dockerfile.kb.dockerignore index b23ce67a5..e4ecd3c26 100644 --- a/deploy_artifacts/Dockerfile.kb.dockerignore +++ b/deploy_artifacts/Dockerfile.kb.dockerignore @@ -15,6 +15,7 @@ **/node_modules ui/dist* ui/.summary-build +ui/.codex-validation* apps/static data logs diff --git a/deploy_artifacts/Dockerfile.mineru.corex b/deploy_artifacts/Dockerfile.mineru.corex new file mode 100644 index 000000000..9d5d13378 --- /dev/null +++ b/deploy_artifacts/Dockerfile.mineru.corex @@ -0,0 +1,33 @@ +# MinerU 2.7.6 / Iluvatar CoreX +FROM crpi-vofi3w62lkohhxsp.cn-shanghai.personal.cr.aliyuncs.com/opendatalab-mineru/corex:4.4.0_torch2.7.1_vllm0.11.2_py3.10 + +ENV MINERU_VLLM_DEVICE=corex \ + DEBIAN_FRONTEND=noninteractive \ + MINERU_MODEL_SOURCE=local \ + MINERU_MODEL_DIR=/models \ + MINERU_TOOLS_CONFIG_JSON=/root/mineru.json \ + MINERU_API_MAX_CONCURRENT_REQUESTS=1 +WORKDIR /app +RUN sed -i 's|http://archive.ubuntu.com/ubuntu|https://mirrors.aliyun.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|https://mirrors.aliyun.com/ubuntu|g' /etc/apt/sources.list && \ + apt-get update && apt-get install -y --no-install-recommends \ + fonts-noto-core fonts-noto-cjk fontconfig libgl1 libglib2.0-0 \ + libreoffice-writer libreoffice-core libcairo2-dev pkg-config && fc-cache -fv && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +COPY . /app +# MinerU branch pins Gradio 5.49.1 (< Pydantic 2.12) while requiring +# Pydantic >=2.12. Gradio 5.50.0 supports Pydantic 2.12 and gradio-pdf <6. +RUN python3 -c "from pathlib import Path; p=Path('pyproject.toml'); s=p.read_text(); old='gradio==5.49.1'; assert s.count(old)==1; p.write_text(s.replace(old, 'gradio==5.50.0'))" +# Resolve against the accelerator wheels already provided by the selected base image. +ARG UV_INDEX_URL +ENV UV_HTTP_TIMEOUT=30 UV_HTTP_RETRIES=2 +RUN python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt && \ + python3 -m pip install 'setuptools>=61,<81' wheel && \ + PIP_INDEX_URL="${UV_INDEX_URL:-https://pypi.org/simple}" python3 -m pip install --no-build-isolation \ + -c /tmp/accelerator-constraints.txt -c docker/constraints-2.7.6.txt '.[core]' 'pycairo==1.26.1' && \ + python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt --check && \ + python3 -m pip check && \ + python3 -c "from mineru.version import __version__; assert __version__ == '2.7.6'" && \ + chmod +x docker/entrypoint.mineru-api.sh && python3 -m pip cache purge +EXPOSE 8000 +ENTRYPOINT ["/app/docker/entrypoint.mineru-api.sh"] +CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/deploy_artifacts/Dockerfile.mineru.cuda b/deploy_artifacts/Dockerfile.mineru.cuda new file mode 100644 index 000000000..08c5e2fea --- /dev/null +++ b/deploy_artifacts/Dockerfile.mineru.cuda @@ -0,0 +1,31 @@ +# MinerU 2.7.6 / NVIDIA CUDA +FROM vllm/vllm-openai:v0.11.2 + +ENV DEBIAN_FRONTEND=noninteractive \ + MINERU_MODEL_SOURCE=local \ + MINERU_MODEL_DIR=/models \ + MINERU_TOOLS_CONFIG_JSON=/root/mineru.json \ + MINERU_API_MAX_CONCURRENT_REQUESTS=1 +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends \ + fonts-noto-core fonts-noto-cjk fontconfig libgl1 libglib2.0-0 \ + libreoffice-writer libreoffice-core libcairo2-dev pkg-config && fc-cache -fv && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +COPY . /app +# MinerU branch pins Gradio 5.49.1 (< Pydantic 2.12) while requiring +# Pydantic >=2.12. Gradio 5.50.0 supports Pydantic 2.12 and gradio-pdf <6. +RUN python3 -c "from pathlib import Path; p=Path('pyproject.toml'); s=p.read_text(); old='gradio==5.49.1'; assert s.count(old)==1; p.write_text(s.replace(old, 'gradio==5.50.0'))" +# Resolve against the accelerator wheels already provided by the selected base image. +ARG UV_INDEX_URL +ENV UV_HTTP_TIMEOUT=30 UV_HTTP_RETRIES=2 +RUN python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt && \ + python3 -m pip install 'setuptools>=61,<81' wheel uv && \ + python3 -m uv pip install --system --no-build-isolation \ + -c /tmp/accelerator-constraints.txt -c docker/constraints-2.7.6.txt '.[core]' 'pycairo==1.26.1' && \ + python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt --check && \ + python3 -m pip check && \ + python3 -c "from mineru.version import __version__; assert __version__ == '2.7.6'" && \ + chmod +x docker/entrypoint.mineru-api.sh && python3 -m pip cache purge +EXPOSE 8000 +ENTRYPOINT ["/app/docker/entrypoint.mineru-api.sh"] +CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/deploy_artifacts/Dockerfile.mineru.cuda.dockerignore b/deploy_artifacts/Dockerfile.mineru.cuda.dockerignore new file mode 100644 index 000000000..adaa09976 --- /dev/null +++ b/deploy_artifacts/Dockerfile.mineru.cuda.dockerignore @@ -0,0 +1,25 @@ +.git +.idea +venv +__pycache__ +*.pyc +*.pyo +*.log +*.tar +*.tar.gz +*.zip +output +outputs +tmp +.cache +node_modules +web_ui/node_modules +web_ui/dist +test_files +tests +build +**/.env +**/.env.* +**/.venv +**/venv* +**/*.egg-info diff --git a/deploy_artifacts/Dockerfile.mineru.cuda.slim b/deploy_artifacts/Dockerfile.mineru.cuda.slim new file mode 100644 index 000000000..7ba8e7823 --- /dev/null +++ b/deploy_artifacts/Dockerfile.mineru.cuda.slim @@ -0,0 +1,26 @@ +# MinerU 2.7.6 on the previously shipped, smaller NVIDIA runtime. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +ENV DEBIAN_FRONTEND=noninteractive \ + MINERU_MODEL_SOURCE=local \ + MINERU_MODEL_DIR=/models \ + MINERU_TOOLS_CONFIG_JSON=/root/mineru.json \ + MINERU_API_MAX_CONCURRENT_REQUESTS=1 +WORKDIR /app +# Remove old application sources before copying the pinned 2.7.6 checkout. +RUN rm -rf /app && mkdir -p /app +COPY . /app +RUN python3 -c "from pathlib import Path; p=Path('pyproject.toml'); s=p.read_text(); old='gradio==5.49.1'; assert s.count(old)==1; p.write_text(s.replace(old, 'gradio==5.50.0'))" +ARG PIP_INDEX_URL +RUN --mount=type=cache,target=/root/.cache/pip \ + python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt && \ + printf 'opencv-python-headless==4.11.0.86\ncupy-cuda12x==13.6.0\n' > /tmp/cuda-slim-constraints.txt && \ + python3 -m pip install --no-build-isolation \ + -c /tmp/accelerator-constraints.txt -c docker/constraints-2.7.6.txt -c /tmp/cuda-slim-constraints.txt '.[core]' 'cupy-cuda12x==13.6.0' && \ + python3 scripts/accelerator_constraints.py /tmp/accelerator-constraints.txt --check && \ + python3 -m pip check && \ + python3 -c "from mineru.version import __version__; assert __version__ == '2.7.6'" && \ + chmod +x docker/entrypoint.mineru-api.sh +EXPOSE 8000 +ENTRYPOINT ["/app/docker/entrypoint.mineru-api.sh"] +CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/deploy_artifacts/Dockerfile.mineru.cuda.slim.dockerignore b/deploy_artifacts/Dockerfile.mineru.cuda.slim.dockerignore new file mode 100644 index 000000000..adaa09976 --- /dev/null +++ b/deploy_artifacts/Dockerfile.mineru.cuda.slim.dockerignore @@ -0,0 +1,25 @@ +.git +.idea +venv +__pycache__ +*.pyc +*.pyo +*.log +*.tar +*.tar.gz +*.zip +output +outputs +tmp +.cache +node_modules +web_ui/node_modules +web_ui/dist +test_files +tests +build +**/.env +**/.env.* +**/.venv +**/venv* +**/*.egg-info diff --git a/deploy_artifacts/README_UPGRADE.md b/deploy_artifacts/README_UPGRADE.md index ba0ab8ea9..f8d56ab4d 100644 --- a/deploy_artifacts/README_UPGRADE.md +++ b/deploy_artifacts/README_UPGRADE.md @@ -1,5 +1,7 @@ # v3.2 / develop → v3.2-refact / develop-refact +> 此文档仅供历史参考。`develop-mineru-2.7.6` 的打包及升级步骤请使用 [README_UPGRADE_276.md](README_UPGRADE_276.md);本文件的 2.7.1 分支、镜像和版本说明不适用于新发布包。 + 本目录是重构版离线升级入口。旧部署材料已归档到 MaxKB `v3.2` 分支的 `deploy_artifacts/`。 升级保留原账号、知识库、文档、向量、应用、聊天记录和配置,不重新导入文档、不批量重建向量。数据库迁移在旧数据的**独立副本**上执行,原目录、原容器和原镜像均保留。需要维护窗口;这不是零停机切换。升级后回退到旧数据时间点,不会自动合并新版产生的数据。 diff --git a/deploy_artifacts/README_UPGRADE_276.md b/deploy_artifacts/README_UPGRADE_276.md new file mode 100644 index 000000000..788c589cf --- /dev/null +++ b/deploy_artifacts/README_UPGRADE_276.md @@ -0,0 +1,146 @@ +# v3.2 / develop 升级到 v3.2-refact / develop-mineru-2.7.6 + +> 本文保留 2026-11-10 旧单文件离线包的详细升级记录。新版分镜像包的构建与交付请使用 `README_发布维护.md`,客户操作请使用 `customer/README_客户部署.md`;本文的 `images.tar` 命令不适用于分镜像包。 + +本文用于准备离线镜像包,并在维护窗口内将旧四容器部署升级到 MaxKB 重构版和 MinerU 2.7.6。升级脚本在独立数据副本上执行数据库迁移,保留旧容器、旧镜像和原数据;升级后新旧数据不再同步,回退不会带回新版产生的数据。脚本和本说明尚未完成本组合的真实镜像构建与升级演练,正式使用前必须在隔离环境完成演练和验收。 + +## 1. 版本与拓扑 + +| 内容 | 升级前 | 升级后 | +| --- | --- | --- | +| 应用源码 | MaxKB `v3.2` | `v3.2-refact` | +| 解析源码 | markitdown `develop` | `develop-mineru-2.7.6`,MinerU 2.7.6 | +| 导图页面 | 独立 `mindmap-web` | MaxKB 工作台,旧 `/mindmap/` 和 9898 重定向至工作台 | +| 容器 | `airag`、`mineru-api`、`mindmap-web`、`unis-gateway` | `airag`、`mineru-api`、`unis-hub-worker`、`unis-gateway` | +| 数据 | `/opt/maxkb` bind mount,内部 PostgreSQL 17 / Redis | 独立副本,仍使用容器内部 PostgreSQL / Redis;worker 与应用共享网络及副本 | + +此升级包不包含独立导图 Web 镜像、大模型服务或模型权重。独立导图页面由 MaxKB 工作台替代;原独立页面浏览器中未保存的内容不自动迁移,应事先导出。MinerU 2.7.6 支持 V2 协议、`/file_parse` 和 `/api/file_parse`;需实际测试所选文档及模型权重。 + +CUDA 打包针对目标分支中的依赖冲突应用镜像内修正:`gradio==5.49.1` 要求 Pydantic `<2.12`,而该分支要求 `>=2.12`,因此构建时将 Gradio 调整为 `5.50.0`(支持 Pydantic 至 2.12.3)。CUDA 基础镜像的 PyGObject 缺少 pycairo,构建时补装并执行 `pip check`。这些调整不修改源码仓库,具体构建文件随包及校验值一起保留;仍需完成解析与页面业务验收。 + +知识库镜像将 Pydantic 固定为 2.12.3,并检查 MCP 依赖所用的接口;较新的 Pydantic 版本移除了该接口,会阻止应用初始化。 + +本次只读核对 2026-11-09 测试机 `10.100.52.118`:四个旧容器运行中;`airag` 的 `/opt/maxkb` 挂载至 `/home/tlw/unis/maxkb-fresh-init-20261109`,PGDATA 位于其中;MinerU 模型挂载 `/home/tlw/unis/mineru-models`;GPU 为单张 NVIDIA RTX 3070 Ti,旧容器的 `CUDA_VISIBLE_DEVICES=0`。这些是检查时的状态,升级前以 `inspect` 和实际宿主机信息为准。源码测试服务与 Docker 应用是不同部署,不要覆盖其目录或共享数据库。 + +## 2. 构建前准备 + +在 **Linux 构建机**准备相邻的 `maxkb/` 和 `markitdown/` 仓库,分别检出目标分支,确认需要发布的改动均已提交、工作区干净。当前材料本身若尚未提交,也需先纳入要构建的 MaxKB 提交;脚本会拒绝脏工作区。构建不自动拉取分支,先按项目流程更新并核对提交号。不要将凭据、模型、虚拟环境、构建缓存或生产数据提交到 Git。 + +```bash +git -C maxkb status --short --branch +git -C maxkb rev-parse HEAD +git -C markitdown status --short --branch +git -C markitdown rev-parse HEAD +docker version +docker compose version +df -h +``` + +需要可联网访问基础镜像和依赖源、可执行 Docker build/save,以及足够的构建缓存、镜像和离线包空间。CUDA 构建引用 `vllm/vllm-openai:v0.11.2`;实际驱动、CUDA 和 GPU 兼容性需在目标机验证。此处的构建脚本不使用旧 `installer/Dockerfile` 中可能复用已有前端 dist 的逻辑,而是重新编译 `/admin/` 和 `/chat/`,写入镜像的 `ui/dist` 和 `apps/static`。 + +## 3. 构建 NVIDIA 离线包 + +```bash +cd /path/to/maxkb +VERSION=refact-276-20261109-1 GPU_FLAVOR=cuda bash deploy_artifacts/build-release.sh +``` + +`VERSION` 必须为每次发布设置新值,输出目录不能已存在。默认输出 `maxkb/deploy_artifacts/releases/refact-276-20261109-1-cuda/`: + +```text +images.tar # KB、gateway、MinerU CUDA 三个带版本的镜像 +upgrade.py # 检查、备份、副本升级与回退 +seed_local_v2_parsers.py # 可选:登记本地三种 V2 解析服务及角色授权 +README_UPGRADE_276.md +SOURCES.txt # 两个仓库的提交号、版本、GPU 类型 +SHA256SUMS.txt +``` + +镜像分别为 `unis-kb:版本`、`unis-gateway:版本`、`unis-mineru:版本-cuda`。解析镜像使用目标分支源码和本发布目录中调整过依赖安装的 CUDA Dockerfile,并附加发布标签;升级脚本校验 GPU 类型、组件标签及 MinerU `2.7.6`。CoreX 可用 `GPU_FLAVOR=corex` 单独构建,不能用 CUDA 验收代替天数卡验收;当前 52.118 应使用 CUDA 包。构建、导出和依赖下载可能耗时较长,失败后不要将不完整目录作为安装包。 + +NVIDIA 升级配置显式使用 `nvidia` runtime 和 GPU 设备请求;目标机须已配置对应 runtime。52.118 已实测只有显式指定该 runtime 时 GPU 预检成功。 + +打包后将整个目录复制到目标机的**新目录**,不要覆盖旧 `deploy_artifacts`、旧镜像文件或旧配置。目标机只需导入三张镜像,`docker load` 不会启动容器: + +```bash +cd /path/to/offline-package +sha256sum -c SHA256SUMS.txt +cat SOURCES.txt +sudo docker load -i images.tar +``` + +## 4. 升级前检查与备份准备 + +要求 Linux、Python 3.10+、GNU tar、Docker Compose v2(支持 `up --wait`)及 root/Docker 权限。确认维护窗口、暂停写入并等待任务结束;保留旧发布包、原容器、旧镜像和原数据,不运行 `down -v`、全局 prune 或递归 chmod。需要足够空间保存原数据归档及独立运行副本。模型目录保留不覆盖,先核实其中 Pipeline / VLM 权重与 MinerU 2.7.6 兼容;目录存在不等于模型可加载。若需更新权重,应在独立目录准备并实测,不要覆盖旧权重后再声称可回退。 + +当前自动脚本只复用旧 `mineru-api` 的模型挂载,不提供另选模型目录的参数。如果旧权重不兼容,先停止自动切换并调整、验证部署方案,不能直接执行以下 `apply`。 + +```bash +sudo python3 upgrade.py inspect +sudo docker ps -a +df -h +``` + +`inspect` 只读校验旧容器、挂载、PostgreSQL 17 和模型目录;如果存在外部 DB/Redis、嵌套数据挂载、额外 PostgreSQL 挂载或其他不支持的拓扑,停止自动流程,先制定独立备份与隔离迁移方案。它会检查其他运行容器和原生 PostgreSQL 是否使用原数据,但不能替代对所有外部写入方、挂载和任务的人工核验。测试机 `/var/lib/postgresql/data` 匿名卷不覆盖实际位于 `/opt/maxkb` 的 PGDATA;以检查输出及实际配置为准。 + +在维护窗口,确认旧容器不再写入后: + +```bash +sudo docker stop --time 330 unis-gateway mindmap-web mineru-api airag +sudo mkdir -p /home/tlw/unis/releases +sudo python3 upgrade.py prepare \ + --release-dir /home/tlw/unis/releases/refact-276-20261109-1-cuda \ + --data-dir /home/tlw/unis/maxkb-refact-276-20261109-1 \ + --kb-image unis-kb:refact-276-20261109-1 \ + --parser-image unis-mineru:refact-276-20261109-1-cuda \ + --gateway-image unis-gateway:refact-276-20261109-1 \ + --gpu cuda +``` + +发布目录和副本目录必须不存在、互不包含,且不能与原数据或模型目录重叠;父目录须预先存在。`prepare` 不启动新容器:它生成私有状态、原容器配置、校验和、数据归档、独立数据副本和固定镜像 ID 的 Compose 配置。发布目录包含敏感环境配置,权限为 0700/0600,不要上传 Git、公开或作为普通安装包分发。 + +如果 `inspect` 显示原 PGDATA 正被其他容器使用,不能对运行中的 PGDATA 直接冷拷贝。仅在确认唯一共享写入方是使用同一 PGDATA 的 PostgreSQL 17 容器时,可经维护窗口评估 `prepare` 的 `--online-db-container 容器名`,由 `pg_basebackup` 生成一致性物理备份;仍需暂停所有业务写入以获得最终切换点。其他共享访问方式须先单独处理,不能强行跳过检查。 + +## 5. 切换与验证 + +```bash +sudo python3 upgrade.py apply \ + --release-dir /home/tlw/unis/releases/refact-276-20261109-1-cuda +``` + +`apply` 核验快照、原容器身份和端口,先在候选 MinerU 镜像中检查版本、Torch/vLLM 和 GPU,再重命名保留旧容器并启动新容器。新 `airag` 的启动流程会对**副本**执行 Django 前向迁移;`apply` 检查迁移后执行 `unify_platform_roles`,在首次创建统一管理员角色时导入旧版 ADMIN 用户,并检查内置 PostgreSQL/Redis。已有统一管理员角色时不会重新导入,以免撤销后又被提权。不得将该操作指向共享原库。失败时新容器会被停止,原容器和副本保留;依据状态执行下节回退。不要将健康检查通过当成业务验收通过。 + +管理员登录后在“解析服务”配置 `http://mineru-api:8000`,选择 **2.7.6**(内部协议为 `legacy`)并测试,再按权限设置工作台默认解析服务和平台模型。旧独立服务的模型环境变量不会自动转换为平台模型,密钥仅在目标机安全配置。确认用户、知识库、文档、向量检索、应用和会话记录与升级前一致。 + +也可选择执行以下增量初始化脚本,先预览,再按需应用;升级和容器启动不会自动执行它: + +```bash +sudo docker exec -i airag python - < seed_local_v2_parsers.py +sudo docker exec -i airag python - --apply < seed_local_v2_parsers.py +``` + +脚本检查本地解析服务的 `/health` 和 2.7.6 版本,登记“传统管道”“混合引擎”“VLM引擎”三种后端,并将各解析资源的查看和使用权限授权给内置管理员及普通用户角色。默认工作区为 `default`,默认地址为 `http://mineru-api:8000`;自定义地址可在 `--endpoint` 后传入。脚本不选择平台默认解析器,也不改变用户其他权限。重复执行不会重复创建;此前由脚本创建的旧名称会按固定资源 ID 原位更新,保留授权。发现同名资源、角色配置或已有授权与预期冲突时,会停止并回滚本次改动。Hybrid 和 VLM 的模型加载、显存和实际解析能力需在目标设备单独验收;资源登记成功不代表 GPU 推理通过。 + +上线前至少检查: + +1. `sudo docker ps -a`;查看容器运行与健康状态;从服务器检查 8989 登录页、注册页、`/chat/`,及 9898、`/mindmap/` 是否进入新版工作台;浏览器确认静态资源没有 404。 +2. 已有账号、权限、知识库问答和实际向量检索正常;业务模型和 Hub worker 可完成真实任务。 +3. 使用标识清晰的测试文档,分别验证 PDF、图片 OCR、Word、标题和表格、总结、保存重开、导出。实际效果、显存及 VLM 能力以目标机结果为准。 +4. 容器内检查 GPU 可见性与实际解析;重建候选容器后验证导图图片和数据持久化。检查源码测试服务未受到影响。 + +必要的故障日志在目标机本地查看,日志可能包含业务数据或凭据,不要直接公开。 + +## 6. 回退 + +先停止新版本业务写入并记录新版产生的数据。以下命令停止本发布的新容器、保留其数据和容器,并恢复旧容器名称;默认不启动旧容器: + +```bash +sudo python3 upgrade.py rollback \ + --release-dir /home/tlw/unis/releases/refact-276-20261109-1-cuda +``` + +如确认原 PGDATA 没有其他服务使用,且接受回到升级前数据时点,可在同一次调用中增加 `--start-old`。旧容器启动前仍须确认端口、原模型和其他服务的实际使用情况。回退不会自动合并新版数据;保留新副本,按业务需要另行导出或补录。若仅需恢复失败发布后的旧名称,使用不带 `--start-old` 的形式。 + +## 7. 已知验证边界 + +本目录的历史 `VALIDATION_20261106.md` 和 `README_UPGRADE.md` 针对旧的 2.7.1 组合,不代表此次 2.7.6 镜像已构建或演练。升级脚本单元测试与 Shell 语法检查不连接 Docker;真实镜像构建、旧版数据库首次前向迁移、模型权重兼容、GPU PDF/OCR 及登录后端到端验收均需单独记录。 diff --git a/deploy_artifacts/README_发布维护.md b/deploy_artifacts/README_发布维护.md new file mode 100644 index 000000000..382760cf0 --- /dev/null +++ b/deploy_artifacts/README_发布维护.md @@ -0,0 +1,27 @@ +# 打包与部署维护 + +客户使用 `customer/README_客户部署.md`;该目录提供分镜像导入及全新部署入口。升级使用 `upgrade.py`,仅支持其检查通过的旧四容器及内置 PostgreSQL 17/Redis 布局。`README_UPGRADE_276.md` 保留详细的旧版升级、备份和回退说明;上线记录见 `DEPLOYED_20261110.md`。 + +## 从源码构建 + +在 Linux 构建机准备相邻的 MaxKB `v3.2-refact` 和 markitdown `develop-mineru-2.7.6`,检查提交和工作树,所需改动必须提交;不要将密码、模型、数据或本机依赖放入上下文。两种构建须使用独立 VERSION,保留日志与 `SOURCES.txt`。例如: + +```bash +VERSION=release-cuda GPU_FLAVOR=cuda bash deploy_artifacts/build-release.sh +VERSION=release-corex GPU_FLAVOR=corex bash deploy_artifacts/build-release.sh +``` + +CoreX 使用 `Dockerfile.mineru.corex`;CUDA 精简版使用 `Dockerfile.mineru.cuda.slim`,默认以已有旧版 `unis-mindmap-api:cuda` 为基础,构建前应从旧交付包导入并核对其镜像 ID,也可以通过 `CUDA_BASE_IMAGE` 指定已核验的镜像 ID。此构建会更新 MinerU 及兼容依赖并检查原有加速器包版本;旧版应用源码会被新分支源码替换。`Dockerfile.mineru.cuda` 保留完整 vLLM 基础镜像的构建方式。 + +构建成功不等于实际加速卡可用:必须分别在目标硬件上验证驱动、模型加载及真实解析。准备好两种镜像后,将公共 KB、网关与两种解析镜像分别导出: + +```bash +ARTIFACT_DIR=/data/releases/customer-package \ +KB_IMAGE=unis-kb:release-cuda GATEWAY_IMAGE=unis-gateway:release-cuda \ +CUDA_IMAGE=unis-mineru:release-cuda-cuda COREX_IMAGE=unis-mineru:release-corex-corex \ +bash deploy_artifacts/package-split.sh +``` + +导出的 `IMAGES.txt` 和 `SHA256SUMS.txt` 与实际镜像一起交付;每个客户只需下载对应设备的解析镜像。模型权重单独交付。禁止把运行环境的 `compose.json`、`state.json`、数据库备份或凭据放进交付包。 + +发布前验证全新部署、升级、角色权限和回退,检查模型权重与硬件;未经验证的 GPU 平台须明确标记。客户包中的新建数据目录与升级副本应与现有数据隔离。 diff --git a/deploy_artifacts/build-compat-release.sh b/deploy_artifacts/build-compat-release.sh new file mode 100644 index 000000000..115ab72c9 --- /dev/null +++ b/deploy_artifacts/build-compat-release.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail +TOOLS_DIR=$(cd "$(dirname "$0")" && pwd) +VERSION=${VERSION:?Set a new release VERSION} +PREVIOUS_VERSION=${PREVIOUS_VERSION:?Set the previously built release VERSION} +ARTIFACT_DIR=${ARTIFACT_DIR:?Set a new output directory} +BUILD_NETWORK=${BUILD_NETWORK:-default} +[[ "$VERSION" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ && "$PREVIOUS_VERSION" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ ]] || exit 2 +test ! -e "$ARTIFACT_DIR" || { echo 'Output directory already exists' >&2; exit 2; } +for image in "unis-kb:$VERSION" "unis-gateway:$VERSION" "unis-mineru:$VERSION-cuda"; do + if docker image inspect "$image" >/dev/null 2>&1; then echo "Target image already exists: $image" >&2; exit 2; fi +done +base_id=$(docker image inspect --format '{{.Id}}' "unis-kb:$PREVIOUS_VERSION") +gateway_id=$(docker image inspect --format '{{.Id}}' "unis-gateway:$PREVIOUS_VERSION") +parser_id=$(docker image inspect --format '{{.Id}}' "unis-mineru:$PREVIOUS_VERSION-cuda") +docker build --network "$BUILD_NETWORK" --build-arg "BASE_IMAGE=unis-kb:$PREVIOUS_VERSION" --build-arg UV_INDEX_URL \ + -f "$TOOLS_DIR/Dockerfile.kb.compat" -t "unis-kb:$VERSION" "$TOOLS_DIR" +test "$(docker image inspect --format '{{.Id}}' "unis-kb:$PREVIOUS_VERSION")" = "$base_id" || { echo 'Base image changed during build' >&2; exit 1; } +docker tag "$gateway_id" "unis-gateway:$VERSION" +docker tag "$parser_id" "unis-mineru:$VERSION-cuda" +mkdir -p "$ARTIFACT_DIR" +docker save -o "$ARTIFACT_DIR/images.tar" "unis-kb:$VERSION" "unis-gateway:$VERSION" "unis-mineru:$VERSION-cuda" +cp "$TOOLS_DIR/upgrade.py" "$TOOLS_DIR/seed_local_v2_parsers.py" "$TOOLS_DIR/README_UPGRADE_276.md" "$TOOLS_DIR/build-compat-release.sh" "$TOOLS_DIR/Dockerfile.kb.compat" "$ARTIFACT_DIR/" +printf 'VERSION=%s\nBASE_VERSION=%s\nKB_BASE_IMAGE_ID=%s\nGATEWAY_IMAGE_ID=%s\nPARSER_IMAGE_ID=%s\n' \ + "$VERSION" "$PREVIOUS_VERSION" "$base_id" "$gateway_id" "$parser_id" > "$ARTIFACT_DIR/BASE_IMAGES.txt" +test -n "${SOURCE_MANIFEST:-}" +awk '!/^(VERSION|GPU_FLAVOR)=/' "$SOURCE_MANIFEST" > "$ARTIFACT_DIR/SOURCES.txt" +printf 'VERSION=%s\nGPU_FLAVOR=cuda\n' "$VERSION" >> "$ARTIFACT_DIR/SOURCES.txt" +(cd "$ARTIFACT_DIR" && sha256sum images.tar upgrade.py seed_local_v2_parsers.py README_UPGRADE_276.md build-compat-release.sh Dockerfile.kb.compat BASE_IMAGES.txt SOURCES.txt > SHA256SUMS.txt) +echo "Compatibility release created: $ARTIFACT_DIR" diff --git a/deploy_artifacts/build-release.sh b/deploy_artifacts/build-release.sh index 092f583fc..0f6f22fa3 100644 --- a/deploy_artifacts/build-release.sh +++ b/deploy_artifacts/build-release.sh @@ -1,12 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -ROOT=$(cd "$(dirname "$0")/.." && pwd) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +ROOT=${MAXKB_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)} +TOOLS_DIR=${RELEASE_TOOLS_DIR:-$ROOT/deploy_artifacts} MINERU_ROOT=${MINERU_ROOT:-$(cd "$ROOT/../markitdown" && pwd)} VERSION=${VERSION:?Set an immutable VERSION, e.g. refact-20261106-1} GPU_FLAVOR=${GPU_FLAVOR:?Set GPU_FLAVOR=cuda or corex} +BUILD_NETWORK=${BUILD_NETWORK:-default} [[ "$VERSION" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ ]] || exit 2 case "$GPU_FLAVOR" in cuda|corex) ;; *) echo 'GPU_FLAVOR must be cuda or corex' >&2; exit 2 ;; esac -for item in "$ROOT:v3.2-refact" "$MINERU_ROOT:develop-refact"; do +for item in "$ROOT:v3.2-refact" "$MINERU_ROOT:develop-mineru-2.7.6"; do repo=${item%:*}; branch=${item##*:} test "$(git -C "$repo" branch --show-current)" = "$branch" || { echo "Wrong source branch: $branch required" >&2; exit 2; } test -z "$(git -C "$repo" status --porcelain --untracked-files=normal)" || { @@ -16,13 +19,23 @@ done OUT=${ARTIFACT_DIR:-$ROOT/deploy_artifacts/releases/$VERSION-$GPU_FLAVOR} test ! -e "$OUT" || { echo 'Artifact directory already exists' >&2; exit 2; } mkdir -p "$OUT" -docker build -f "$ROOT/deploy_artifacts/Dockerfile.kb" --build-arg RELEASE_VERSION="$VERSION" \ +docker build --network "$BUILD_NETWORK" -f "$TOOLS_DIR/Dockerfile.kb" --build-arg UV_INDEX_URL --build-arg RELEASE_VERSION="$VERSION" \ --build-arg GITHUB_COMMIT="$(git -C "$ROOT" rev-parse HEAD)" -t "unis-kb:$VERSION" "$ROOT" docker build -f "$ROOT/deploy_artifacts/gateway/Dockerfile" -t "unis-gateway:$VERSION" "$ROOT/deploy_artifacts/gateway" -API_FLAVOR="$GPU_FLAVOR" VERSION="$VERSION" bash "$MINERU_ROOT/scripts/build-images.sh" -docker save -o "$OUT/images.tar" "unis-kb:$VERSION" "unis-gateway:$VERSION" "unis-mineru:$VERSION-$GPU_FLAVOR" -cp "$ROOT/deploy_artifacts/upgrade.py" "$ROOT/deploy_artifacts/README_UPGRADE.md" "$OUT/" +case "$GPU_FLAVOR" in + cuda) parser_dockerfile="$TOOLS_DIR/Dockerfile.mineru.cuda.slim" ;; + corex) parser_dockerfile="$TOOLS_DIR/Dockerfile.mineru.corex" ;; +esac +docker build --network "$BUILD_NETWORK" -f "$parser_dockerfile" --build-arg UV_INDEX_URL --build-arg PIP_INDEX_URL --build-arg "BASE_IMAGE=${CUDA_BASE_IMAGE:-unis-mindmap-api:cuda}" \ + --label io.unis.component=parser --label io.unis.layout=refact \ + --label "io.unis.gpu=$GPU_FLAVOR" \ + --label "org.opencontainers.image.revision=$(git -C "$MINERU_ROOT" rev-parse HEAD)" \ + -t "unis-mineru:$VERSION-$GPU_FLAVOR" "$MINERU_ROOT" +docker save "unis-kb:$VERSION" | gzip -1 > "$OUT/unis-kb.tar.gz" +docker save "unis-gateway:$VERSION" | gzip -1 > "$OUT/unis-gateway.tar.gz" +docker save "unis-mineru:$VERSION-$GPU_FLAVOR" | gzip -1 > "$OUT/unis-mineru-$GPU_FLAVOR.tar.gz" +cp "$TOOLS_DIR/upgrade.py" "$TOOLS_DIR/seed_local_v2_parsers.py" "$TOOLS_DIR/README_UPGRADE_276.md" "$TOOLS_DIR/build-release.sh" "$TOOLS_DIR/Dockerfile.kb" "$TOOLS_DIR/Dockerfile.kb.dockerignore" "$TOOLS_DIR/Dockerfile.mineru.cuda.slim" "$TOOLS_DIR/Dockerfile.mineru.cuda.slim.dockerignore" "$TOOLS_DIR/Dockerfile.mineru.corex" "$OUT/" printf 'MAXKB_COMMIT=%s\nMINERU_COMMIT=%s\nVERSION=%s\nGPU_FLAVOR=%s\n' \ "$(git -C "$ROOT" rev-parse HEAD)" "$(git -C "$MINERU_ROOT" rev-parse HEAD)" "$VERSION" "$GPU_FLAVOR" > "$OUT/SOURCES.txt" -(cd "$OUT" && sha256sum images.tar upgrade.py README_UPGRADE.md SOURCES.txt > SHA256SUMS.txt) +(cd "$OUT" && sha256sum unis-*.tar.gz upgrade.py seed_local_v2_parsers.py README_UPGRADE_276.md build-release.sh Dockerfile.kb Dockerfile.kb.dockerignore Dockerfile.mineru.cuda.slim Dockerfile.mineru.cuda.slim.dockerignore Dockerfile.mineru.corex SOURCES.txt > SHA256SUMS.txt) echo "Offline release created: $OUT" diff --git a/deploy_artifacts/customer/README_客户部署.md b/deploy_artifacts/customer/README_客户部署.md new file mode 100644 index 000000000..1aa5a9649 --- /dev/null +++ b/deploy_artifacts/customer/README_客户部署.md @@ -0,0 +1,49 @@ +# 部署说明 + +准备 Docker、Docker Compose、对应 GPU 驱动和 MinerU 2.7.6 模型文件。模型目录需包含 `pipeline` 和 `vlm` 子目录。根据设备选择 `cuda`(NVIDIA)或 `corex`(天数);只需下载公共的 `unis-kb.tar.gz`、`unis-gateway.tar.gz` 和对应的 `unis-mineru-<类型>.tar.gz`,以及本目录的脚本、`IMAGES.txt` 和 `SHA256SUMS.txt`。 + +CoreX 版本需在目标天数设备完成启动、模型加载和实际解析验证后使用。 + +```bash +cd /path/to/package +GPU=cuda # 天数设备改成 corex +sha256sum -c --ignore-missing SHA256SUMS.txt +sudo bash load-images.sh "$GPU" +source IMAGES.txt +if [ "$GPU" = cuda ]; then PARSER_IMAGE="$CUDA_IMAGE"; else PARSER_IMAGE="$COREX_IMAGE"; fi +``` + +## 全新部署 + +全新部署会创建空数据库;`--data-dir` 和 `--release-dir` 必须是尚不存在的新目录。请先创建它们的父目录,模型目录使用已有的独立目录。 + +```bash +sudo python3 fresh_install.py --gpu "$GPU" \ + --data-dir /data/unis/kb-new --model-dir /data/unis/mineru-models \ + --release-dir /data/unis/release-new \ + --kb-image "$KB_IMAGE" --gateway-image "$GATEWAY_IMAGE" --parser-image "$PARSER_IMAGE" +``` + +打开 `http://服务器IP:8989/`。部署配置含敏感信息,请妥善保管。 + +## 从旧版升级 + +适用于原四容器、内置 PostgreSQL 17 和 Redis 的部署。维护窗口暂停业务并备份,按现场数据库运维流程正常关闭 PostgreSQL;如检查提示外部数据库或共享数据,请停止操作并联系部署人员。 + +```bash +sudo python3 upgrade.py inspect +sudo docker stop --time 330 unis-gateway mindmap-web mineru-api airag +sudo python3 upgrade.py prepare --gpu "$GPU" \ + --release-dir /data/unis/release-upgrade --data-dir /data/unis/kb-upgrade \ + --kb-image "$KB_IMAGE" --gateway-image "$GATEWAY_IMAGE" --parser-image "$PARSER_IMAGE" +sudo python3 upgrade.py apply --release-dir /data/unis/release-upgrade +``` + +升级使用独立数据副本,保留原数据。升级后验证账号、知识库和解析功能;必要时执行 `sudo python3 upgrade.py rollback --release-dir /data/unis/release-upgrade --start-old` 回退。回退不会自动合并升级后新增的数据。 + +如需创建三种本地解析工具及授权,可预览后执行: + +```bash +sudo docker exec -i airag python - < seed_local_v2_parsers.py +sudo docker exec -i airag python - --apply < seed_local_v2_parsers.py +``` diff --git a/deploy_artifacts/customer/fresh_install.py b/deploy_artifacts/customer/fresh_install.py new file mode 100644 index 000000000..248dbe99f --- /dev/null +++ b/deploy_artifacts/customer/fresh_install.py @@ -0,0 +1,69 @@ +"""Create an isolated, new data directory and start the release containers.""" +import argparse +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys + +import upgrade + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--gpu', choices=['cuda', 'corex'], required=True) + for name in ('data-dir', 'model-dir', 'release-dir', 'kb-image', 'gateway-image', 'parser-image'): + p.add_argument('--' + name, required=True) + a = p.parse_args() + os.umask(0o077) + data, models, release = [upgrade.safe_path(getattr(a, n)) for n in ('data_dir', 'model_dir', 'release_dir')] + if any(upgrade.overlaps(x, y) for i, x in enumerate((data, models, release)) for y in (data, models, release)[i+1:]): + raise ValueError('数据、模型和部署目录必须相互独立') + if data.exists() or release.exists() or not data.parent.is_dir() or not release.parent.is_dir(): + raise ValueError('数据和部署目录必须为不存在的新目录,且其父目录必须存在') + if not all((models / name).is_dir() for name in ('pipeline', 'vlm')): + raise ValueError('模型目录须包含 pipeline 和 vlm 子目录') + for name in upgrade.NEW: + if subprocess.run(['docker', 'inspect', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0: + raise ValueError('目标容器名称已存在:' + name) + images = {} + for component, name in [('kb', a.kb_image), ('gateway', a.gateway_image), ('parser', a.parser_image)]: + row = json.loads(upgrade.run('docker', 'image', 'inspect', name))[0] + labels = row['Config'].get('Labels') or {} + if labels.get('io.unis.layout') != 'refact' or labels.get('io.unis.component') != component: + raise ValueError('镜像类型不匹配:' + component) + if component == 'parser' and labels.get('io.unis.gpu') != a.gpu: + raise ValueError('解析镜像与 GPU 类型不匹配') + images[component] = row['Id'] + db_password, redis_password = secrets.token_hex(24), secrets.token_hex(24) + env = {'MAXKB_CONFIG_TYPE': 'ENV', 'MAXKB_DB_NAME': 'maxkb', 'MAXKB_DB_USER': 'postgres', + 'MAXKB_DB_PASSWORD': db_password, 'POSTGRES_USER': 'postgres', 'POSTGRES_PASSWORD': db_password, + 'MAXKB_REDIS_PASSWORD': redis_password, 'REDIS_PASSWORD': redis_password, 'MAXKB_REDIS_DB': '1', + 'MAXKB_DB_HOST': '127.0.0.1', 'MAXKB_REDIS_HOST': '127.0.0.1', + 'PGDATA': '/opt/maxkb/data/postgresql/pgdata'} + def row(environment=None, port=None): + return {'Config': {'Env': [k + '=' + v for k, v in (environment or {}).items()]}, + 'HostConfig': {'PortBindings': {str(port) + '/tcp': [{'HostIp': '0.0.0.0', 'HostPort': str(port)}]} if port else {}}} + rows = {'airag': row(env), 'mineru-api': row(), 'unis-gateway': row(port=8989), 'mindmap-web': row(port=9898)} + config = upgrade.make_compose(rows, images, data, models, a.gpu, 'unis-fresh') + # PostgreSQL runs as a non-root user and must be able to traverse this mount. + data.mkdir(mode=0o755); data.chmod(0o755); (data / 'mineru-output').mkdir() + release.mkdir(mode=0o700) + upgrade.save(release / 'compose.json', upgrade.escape_compose(config)) + upgrade.compose(release, 'config', '--quiet') + upgrade.compose(release, 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '360') + upgrade.run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') + # Create the unified administrator role and attach the freshly initialized admin. + upgrade.run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles') + upgrade.run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py') + print('部署完成。请打开 http://服务器IP:8989/ 并完成业务验证。') + print('部署配置位于', release, ',含敏感信息,请妥善保管。') + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, KeyError) as exc: + print('部署停止:' + (str(exc) if isinstance(exc, (ValueError, RuntimeError)) else type(exc).__name__), file=sys.stderr) + sys.exit(1) diff --git a/deploy_artifacts/customer/load-images.sh b/deploy_artifacts/customer/load-images.sh new file mode 100644 index 000000000..39f8ae925 --- /dev/null +++ b/deploy_artifacts/customer/load-images.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +case "${1:-}" in cuda|corex) flavor=$1 ;; *) echo '用法: bash load-images.sh cuda|corex' >&2; exit 2 ;; esac +for file in unis-kb.tar.gz unis-gateway.tar.gz "unis-mineru-${flavor}.tar.gz"; do + test -f "$file" || { echo "缺少文件: $file" >&2; exit 1; } + sha256sum --check --ignore-missing <(grep -F " $file" SHA256SUMS.txt) + docker load -i "$file" +done diff --git a/deploy_artifacts/package-split.sh b/deploy_artifacts/package-split.sh new file mode 100644 index 000000000..d821622d8 --- /dev/null +++ b/deploy_artifacts/package-split.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +tools=$(cd "$(dirname "$0")" && pwd) +: "${ARTIFACT_DIR:?Set ARTIFACT_DIR to a new directory}" +: "${KB_IMAGE:?Set KB_IMAGE}" +: "${GATEWAY_IMAGE:?Set GATEWAY_IMAGE}" +: "${CUDA_IMAGE:?Set CUDA_IMAGE}" +: "${COREX_IMAGE:?Set COREX_IMAGE}" +test ! -e "$ARTIFACT_DIR" || { echo 'Output directory already exists' >&2; exit 1; } +for image in "$KB_IMAGE" "$GATEWAY_IMAGE" "$CUDA_IMAGE" "$COREX_IMAGE"; do docker image inspect "$image" >/dev/null; done +mkdir -m 700 "$ARTIFACT_DIR" +save() { docker save "$1" | gzip -1 > "$ARTIFACT_DIR/$2.part"; mv "$ARTIFACT_DIR/$2.part" "$ARTIFACT_DIR/$2"; } +save "$KB_IMAGE" unis-kb.tar.gz +save "$GATEWAY_IMAGE" unis-gateway.tar.gz +save "$CUDA_IMAGE" unis-mineru-cuda.tar.gz +save "$COREX_IMAGE" unis-mineru-corex.tar.gz +cp "$tools"/upgrade.py "$tools"/seed_local_v2_parsers.py "$tools"/customer/* "$ARTIFACT_DIR/" +printf 'KB_IMAGE=%s\nGATEWAY_IMAGE=%s\nCUDA_IMAGE=%s\nCOREX_IMAGE=%s\n' "$KB_IMAGE" "$GATEWAY_IMAGE" "$CUDA_IMAGE" "$COREX_IMAGE" > "$ARTIFACT_DIR/IMAGES.txt" +for image in "$KB_IMAGE" "$GATEWAY_IMAGE" "$CUDA_IMAGE" "$COREX_IMAGE"; do + docker image inspect --format '{{.Id}} {{index .Config.Labels "org.opencontainers.image.revision"}}' "$image" +done > "$ARTIFACT_DIR/IMAGE_IDS.txt" +(cd "$ARTIFACT_DIR" && sha256sum unis-*.tar.gz upgrade.py seed_local_v2_parsers.py fresh_install.py load-images.sh README_客户部署.md IMAGES.txt IMAGE_IDS.txt > SHA256SUMS.txt) diff --git a/deploy_artifacts/seed_local_v2_parsers.py b/deploy_artifacts/seed_local_v2_parsers.py new file mode 100644 index 000000000..b32d33c22 --- /dev/null +++ b/deploy_artifacts/seed_local_v2_parsers.py @@ -0,0 +1,130 @@ +"""Optionally register local MinerU 2.7.6 backends and grant two built-in roles. + +Run in the MaxKB container; preview is read-only unless --apply is passed. +Existing conflicting resources/grants cause a transaction rollback, never overwrite. +""" +import argparse +import json +import os +import sys +from urllib.parse import urlsplit +from urllib.request import urlopen +import uuid + +APP_ROOT = '/opt/maxkb-app' +sys.path.insert(0, APP_ROOT + '/apps') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + +import django +django.setup() + +from django.db import transaction +from system_manage.ai_hub import parser_save +from system_manage.models.ai_hub import AIParserResource +from system_manage.models.platform_access import PlatformGrant, PlatformRole +from system_manage.parsers.catalog import validate_options +from system_manage.platform_access import ADMIN_ROLE_ID, DEFAULT_ROLE_ID, BUILTIN_ADMIN_USER_ID, is_admin +from users.models import User + +WORKSPACE = 'default' +VERSION = '2.7.6' +RESOURCES = ( + ('pipeline', '传统管道', 'pipeline', '本地 V2 - Pipeline'), + ('hybrid', '混合引擎', 'hybrid-auto-engine', '本地 V2 - Hybrid'), + ('vlm', 'VLM引擎', 'vlm-auto-engine', '本地 V2 - VLM'), +) +ROLES = (ADMIN_ROLE_ID, DEFAULT_ROLE_ID) + + +def resource_id(key): + return uuid.uuid5(uuid.NAMESPACE_URL, 'unis:local-parser:v2:2.7.6:' + key) + + +def endpoint_value(value): + value = value.rstrip('/') + parsed = urlsplit(value) + if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError('解析服务地址无效') + return value + + +def verify_health(endpoint): + with urlopen(endpoint + '/health', timeout=10) as response: + payload = json.load(response) + if not isinstance(payload, dict) or str(payload.get('version')) != VERSION: + raise ValueError('解析服务健康检查失败或版本不是 2.7.6') + + +def seed(endpoint, apply): + endpoint = endpoint_value(endpoint) + verify_health(endpoint) + result = {'mode': 'apply' if apply else 'preview', 'version': VERSION, 'resources': [], 'grants_created': 0, 'grants_to_create': 0} + with transaction.atomic(): + # Serialize concurrent runs and fail before any write if built-in roles are unavailable. + roles = list(PlatformRole.objects.select_for_update().filter(id__in=ROLES).order_by('id')) + if {str(role.id) for role in roles} != set(ROLES) or any(not role.enabled for role in roles): + raise ValueError('管理员或普通用户角色不存在或已停用') + default = next(role for role in roles if str(role.id) == DEFAULT_ROLE_ID) + if not default.is_default: + raise ValueError('普通用户角色不是默认角色') + admin = User.objects.filter(id=BUILTIN_ADMIN_USER_ID, is_active=True).first() + if not admin or not is_admin(admin): + raise ValueError('内置管理员账号不存在、停用或尚未授予统一管理员角色') + for key, name, backend, previous_name in RESOURCES: + pk = resource_id(key) + options = validate_options({'protocol': 'legacy', 'engine_version': VERSION, + 'backend': backend, 'language': 'ch'}, endpoint) + obj = AIParserResource.objects.filter(pk=pk).first() + existed = obj is not None + if obj: + if (obj.workspace_id != WORKSPACE or obj.name not in (name, previous_name) or obj.endpoint != endpoint + or obj.options != options or not obj.enabled or obj.credential): + raise ValueError('已有初始化资源与预期不一致:' + name) + if AIParserResource.objects.filter(workspace_id=WORKSPACE, name=name).exclude(pk=pk).exists(): + raise ValueError('已存在同名解析资源,请先人工核对:' + name) + rename = bool(obj and obj.name == previous_name) + grants = [] + for role_id in ROLES: + grant = PlatformGrant.objects.filter(workspace_id=WORKSPACE, subject_type='ROLE', + subject_id=role_id, resource_type='PARSER', target=str(pk)).first() + if grant and ('USE' not in (grant.actions or []) or grant.expires_at is not None): + raise ValueError('已有角色授权与预期不一致,请先人工核对:' + name) + grants.append((role_id, grant)) + if grant is None: + result['grants_to_create'] += 1 + if apply and obj is None: + obj = AIParserResource(id=pk, workspace_id=WORKSPACE, owner_id=admin.id, + name=name, endpoint=endpoint) + parser_save(WORKSPACE, admin, {'name': name, 'endpoint': endpoint, 'options': options}, obj) + elif apply and rename: + parser_save(WORKSPACE, admin, {'name': name}, obj) + for role_id, grant in grants: + if apply and grant is None: + PlatformGrant.objects.create(workspace_id=WORKSPACE, subject_type='ROLE', + subject_id=role_id, resource_type='PARSER', target=str(pk), + actions=['VIEW', 'USE'], granted_by_id=admin.id) + result['grants_created'] += 1 + result['resources'].append({'name': name, 'backend': backend, + 'action': ('renamed' if apply else 'rename') if rename else + ('existing' if existed else ('created' if apply else 'create'))}) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--endpoint', default='http://mineru-api:8000', help='MinerU 2.7.6 API base URL') + parser.add_argument('--apply', action='store_true', help='Create the resources and role grants') + args = parser.parse_args() + print(json.dumps(seed(args.endpoint, args.apply), ensure_ascii=False)) + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + # Avoid dumping connection details or private configuration. + if isinstance(exc, ValueError): + print('初始化失败:' + str(exc), file=sys.stderr) + else: + print('初始化失败:' + type(exc).__name__, file=sys.stderr) + sys.exit(1) diff --git a/deploy_artifacts/tests/test_upgrade.py b/deploy_artifacts/tests/test_upgrade.py index 64b606096..09e2f1770 100644 --- a/deploy_artifacts/tests/test_upgrade.py +++ b/deploy_artifacts/tests/test_upgrade.py @@ -41,6 +41,7 @@ class UpgradeTests(unittest.TestCase): cuda = self.config()['services']['mineru-api'] corex = self.config('corex')['services']['mineru-api'] self.assertEqual(cuda['deploy']['resources']['reservations']['devices'][0]['driver'], 'nvidia') + self.assertEqual(cuda['runtime'], 'nvidia') self.assertNotIn('privileged', cuda) self.assertNotIn('runtime', corex) self.assertNotIn('deploy', corex) @@ -227,6 +228,22 @@ class UpgradeTests(unittest.TestCase): self.assertTrue(all(call.args[:2] == ('docker', 'rename') for call in runner.call_args_list)) self.assertEqual(compose.call_args.args[1], 'stop') + def test_apply_initializes_unified_roles_after_migrations(self): + rows = containers() + state = {'phase': 'prepared', 'project': 'this-release', 'data': '/srv/unis/clone', 'old': rows, + 'original_signature': 'same', 'clone_signature': 'same', 'renamed': [], 'new_ids': {}} + def inspect_new(name): + return {'Id': 'new-' + name, 'Config': {'Labels': {'com.docker.compose.project': 'this-release'}}} + def run_command(*args): + return '{}' if args[-1] == '/usr/local/bin/unis-check-internal-services.py' else '' + with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='same'), patch.object(upgrade.socket, 'socket'), patch.object(upgrade, 'save'), patch.object(upgrade, 'compose'), patch.object(upgrade, 'inspect', side_effect=inspect_new), patch.object(upgrade, 'run', side_effect=run_command) as runner: + upgrade.apply(Path('/srv/unis/release')) + commands = [call.args for call in runner.call_args_list] + migration = ('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') + roles = ('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles') + self.assertLess(commands.index(migration), commands.index(roles)) + self.assertEqual(state['phase'], 'started-awaiting-acceptance') + def test_online_backup_container_must_mount_exact_legacy_pgdata(self): row = {'Id': 'shared-pg', 'State': {'Running': True}, 'Config': {'Env': ['PG_MAJOR=17', 'PGDATA=/var/lib/postgresql/data']}, diff --git a/deploy_artifacts/upgrade.py b/deploy_artifacts/upgrade.py index 9c80835ff..bec8d7ccb 100644 --- a/deploy_artifacts/upgrade.py +++ b/deploy_artifacts/upgrade.py @@ -178,8 +178,9 @@ def make_compose(rows, images, data, models, flavor, project): 'environment': parser_env, 'volumes': [ {'type': 'bind', 'source': str(models), 'target': '/models', 'read_only': True, 'bind': {'create_host_path': False}}, {'type': 'bind', 'source': str(data / 'mineru-output'), 'target': '/app/output', 'bind': {'create_host_path': False}}], - 'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/openapi.json", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}} + 'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}} if flavor == 'cuda': + parser['runtime'] = 'nvidia' parser['deploy'] = {'resources': {'reservations': {'devices': [{'driver': 'nvidia', 'count': 'all', 'capabilities': ['gpu']}]}}} else: # Preserve the established CoreX device exposure; never select NVIDIA runtime. @@ -348,7 +349,7 @@ def apply(release): sock.bind((binding['host_ip'], int(binding['published']))) # No model is loaded in this smoke test. A driver/runtime mismatch fails before switchover. compose(release, 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'python3', 'mineru-api', '-c', - 'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.1"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))') + 'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.6"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))') state['phase'] = 'switching' save(release / 'state.json', state) try: @@ -368,6 +369,10 @@ def apply(release): raise ValueError('Unexpected container identity') state['new_ids'][name] = row['Id'] run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') + # Migration 0008 creates the tables, but the one-time importer is a + # separate command. It imports legacy ADMIN users only on first creation + # of the unified admin role and never restores later demotions. + run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles') checks = run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py') save(release / 'internal-services.json', json.loads(checks.strip().splitlines()[-1])) state['phase'] = 'started-awaiting-acceptance'