1180 lines
56 KiB
Python
1180 lines
56 KiB
Python
#!/usr/bin/env python3
|
||
"""老四算(unis_fourcal_dev)→ 四算 PMS(fourcal_pms_*)的数据导入。
|
||
|
||
设计前提(很重要,改这个脚本前先读)
|
||
====================================
|
||
|
||
* **老库只读**。legacy 连接一开就 `START TRANSACTION READ ONLY`,脚本里也只发 SELECT。
|
||
老库还要继续跑生产,写坏了没人担得起。
|
||
* **默认干跑**。不加 `--apply` 只统计不落库,先看清要动多少数据再真写。
|
||
* **不保留老库主键**。新库自己发号,老 id → 新 id 用内存映射表串起来。
|
||
老 id 直接搬会和 seed 出来的 admin / 演示项目撞号,撞了就是静默覆盖,比报错更糟。
|
||
* **审批历史也不迁**(目前没有对应开关,别去找 `--with-history`,它不存在)。老流程实例
|
||
在 Activiti 里,硬迁过来只能造出一堆来路不明的待办;切流前把在途单子清干净,新送审一律
|
||
走新流程。已经走完的历史审批要看得见的,源数据在老库
|
||
``act_hi_procinst``(2,040 条)/ ``act_hi_taskinst``(9,341 条)/ ``act_hi_comment``
|
||
(17,419 条)里,要做的是把这三张表折成 ``approval_record`` 流水,跟新流程完全解耦;
|
||
这件事得先定"历史流水在新界面里怎么摆",不适合在导数脚本里顺手拍板。
|
||
|
||
覆盖范围
|
||
========
|
||
|
||
org dept → org_unit
|
||
user sys_user → accounts + account_identity + org_position
|
||
role sys_role → perm_role(+ 老用户挂的角色关系)
|
||
project project → project + biz_revision(概算/预算各一版)
|
||
estimate project_estimate_* → estimate_income/cost/other_line
|
||
budget project_budget_* → budget_*_line + budget_fund_month(正式表;_temp 草稿可用 --with-temp)
|
||
flow act_task_def → approval_flow + approval_node
|
||
visible project_visible type=2 → project_member(viewer)
|
||
(type=2 是按人员授权,tid=sys_user.id;type=1 才是按角色,
|
||
tid=sys_role.id → project_visible_role,老库里现存的 15,302 行全是 type=2)
|
||
|
||
归一化
|
||
======
|
||
|
||
老库有几处口径和新系统不一样,脚本会显式转换并在报告里计数,不静默改数:
|
||
|
||
* **税率**:`tax_rate` 老库存小数(`0.13`)→ 百分数(`13`),只放大 (0, 1]。
|
||
* **利润率**:`*_margin` 老库**本来就是百分数**(实测 1578.60/12990 = 12.15),原样搬运,
|
||
不再乘 100。库里真有 0.03 这种值,那是 0.03%,不是 3%。
|
||
* **产品归类**:老 `category` 存的是 `procurement_type` 的 **id 字符串**("8"→华智产品),
|
||
先查字典拿中文名,再折成产品归属编码(`service.calc.mapping`)。
|
||
* **月份**:老库 `start_date`/`end_date` 是 datetime,新系统按 `YYYY-MM` 存,
|
||
优先取 `plan_start_str`/`plan_end_str`,没有才从 datetime 格式化。
|
||
|
||
用法
|
||
====
|
||
|
||
# 先干跑,看数量
|
||
python3 scripts/migrate_legacy.py
|
||
|
||
# 只导组织和人员,先验证头寸
|
||
python3 scripts/migrate_legacy.py --apply --steps org,user,role
|
||
|
||
# 全量落库,前 50 个项目试水
|
||
python3 scripts/migrate_legacy.py --apply --limit-projects 50
|
||
|
||
# 全量
|
||
python3 scripts/migrate_legacy.py --apply
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
from datetime import date
|
||
from decimal import Decimal, InvalidOperation
|
||
from pathlib import Path
|
||
|
||
# 允许从 backend 根目录直接跑这个文件
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
from sqlalchemy import event, inspect, select, text # noqa: E402
|
||
from sqlalchemy.orm import Session # noqa: E402
|
||
|
||
from app.config import settings # noqa: E402
|
||
from app.db import SessionFactory, legacy_engine # noqa: E402
|
||
from app.models.approval import ApprovalFlow, ApprovalNode, NodeType, SignType # noqa: E402
|
||
from app.models.base import Base # noqa: E402
|
||
from app.models.finance import ( # noqa: E402
|
||
BudgetFundMonth,
|
||
BudgetIncomeLine,
|
||
BudgetIncomePlanLine,
|
||
BudgetManageLine,
|
||
BudgetPayPlanLine,
|
||
BudgetPurchaseLine,
|
||
EstimateCostLine,
|
||
EstimateIncomeLine,
|
||
EstimateOtherLine,
|
||
Revision,
|
||
RevisionStatus,
|
||
)
|
||
from app.models.org import Account, AccountIdentity, OrgUnit, Position, Status # noqa: E402
|
||
from app.models.permission import PermAccountRole, PermRole # noqa: E402
|
||
from app.models.project import ( # noqa: E402
|
||
Project, ProjectMember, ProjectMemberRole, ProjectSource, ProjectStage, ProjectVisibleRole,
|
||
SaleStage)
|
||
from app.security import hash_password # noqa: E402
|
||
from app.service.calc.mapping import ( # noqa: E402
|
||
LEGACY_CATEGORY_ALIASES,
|
||
LEGACY_PROCUREMENT_TYPES,
|
||
)
|
||
|
||
ZERO = Decimal("0")
|
||
|
||
# 只在一次运行里缓存兜底账号 id
|
||
_FALLBACK: dict[str, int | None] = {}
|
||
|
||
# 老 act_task_def.task_type 只有 0/1,语义是"任一人"与"全部"
|
||
LEGACY_SIGN_TYPES = {0: SignType.ANY, 1: SignType.ALL}
|
||
|
||
# 老 project_estimate_income.type / project_estimate_cost.type 的中文名,
|
||
# 老库里这两张明细表**没有名称列**,只有类型,所以名称只能按类型生成
|
||
ESTIMATE_INCOME_NAMES = {1: "设备收入", 2: "工程收入", 3: "服务收入"}
|
||
ESTIMATE_COST_NAMES = {1: "设备采购", 2: "施工采购", 3: "服务采购", 4: "其他采购"}
|
||
|
||
# 老 project 的列在新表里基本都有对应,逐列显式赋值(见 migrate_projects)。
|
||
# 刻意不做"同名列自动搬运":新表里 value_risk、main_contract_* 这些是 TEXT,
|
||
# 自动搬运时顺手截个 500 字就是静默丢数据,不如一列一列写清楚。
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 超长字符串兜底
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: "表.列 -> 被截断的行数",跑完由 main() 折进报告
|
||
CLIPPED: dict[str, int] = {}
|
||
|
||
#: 落在四个产品归属之外的老采购类别 -> 行数。按行数排序取前几名进报告,
|
||
#: 别把 40 个只出现一次的自由文本一条条印出来刷屏。
|
||
UNMAPPED_CATEGORY: dict[str, int] = {}
|
||
|
||
|
||
def _clip_oversize_strings(target: object, args: object = None,
|
||
kwargs: object = None, *_: object) -> None:
|
||
"""字符串超过列宽时截断并计数,而不是让整批导入死在 1406 上。
|
||
|
||
列宽从 SQLAlchemy 元数据现读,模型改了这里自动跟上,不用两头维护。
|
||
TEXT 列(length 为 None)一律不动:那种列本来装得下,截断才是真丢数据。
|
||
|
||
两头都截:SQLAlchemy 的 init 事件在声明式构造器**给属性赋值之前**就派发了
|
||
(实测此时 ``target.__dict__`` 还是空的),所以构造参数 kwargs 才是主战场;
|
||
``__dict__`` 那一路留着,万一事件派发顺序变了也不会漏。
|
||
|
||
这是兜底,不是常规手段——列宽该对齐老库的就该改模型(比如 pay_way 从 64
|
||
放到 255,老库那列装的是合同付款条款,实测 136 字)。真触发这里了,报告里会
|
||
写明是哪一列截了几行:要么加宽列,要么确认这段文本确实可以丢。
|
||
"""
|
||
mapper = inspect(type(target), raiseerr=False)
|
||
if mapper is None:
|
||
return
|
||
limits: dict[str, tuple[str, int]] = {}
|
||
for attr in mapper.column_attrs:
|
||
column = attr.columns[0]
|
||
limit = getattr(column.type, "length", None)
|
||
if limit:
|
||
limits[attr.key] = (column.name, int(limit))
|
||
if not limits:
|
||
return
|
||
|
||
# 长度按字符算,和 MySQL utf8mb4 的 VARCHAR(n) 口径一致
|
||
def clip(key: str, value: object) -> object:
|
||
if not isinstance(value, str):
|
||
return value
|
||
bound = limits.get(key)
|
||
if bound is None or len(value) <= bound[1]:
|
||
return value
|
||
column_name, limit = bound
|
||
CLIPPED[f"{mapper.local_table.name}.{column_name}"] = \
|
||
CLIPPED.get(f"{mapper.local_table.name}.{column_name}", 0) + 1
|
||
return value[:limit]
|
||
|
||
if isinstance(kwargs, dict):
|
||
for key in list(kwargs):
|
||
kwargs[key] = clip(key, kwargs[key])
|
||
state = getattr(target, "__dict__", None)
|
||
if isinstance(state, dict):
|
||
for key in list(state):
|
||
state[key] = clip(key, state[key])
|
||
|
||
|
||
def install_clip_guard() -> None:
|
||
"""挂上超长字符串兜底。
|
||
|
||
刻意放在函数里而不是 @event.listens_for 装饰在模块上:模块级的话,
|
||
只要 import 这个脚本,监听器就全局生效,会把同进程里其他东西(比如 pytest
|
||
里的接口测试)建的 ORM 对象一起截了。导数入口显式调一次就够了。
|
||
"""
|
||
if not event.contains(Base, "init", _clip_oversize_strings):
|
||
event.listen(Base, "init", _clip_oversize_strings, propagate=True)
|
||
|
||
|
||
class Report:
|
||
"""导数报告:做了什么、跳过了什么、换算了多少,最后一次性摊开。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.counts: dict[str, int] = {}
|
||
self.skips: dict[str, int] = {}
|
||
self.notes: list[str] = []
|
||
|
||
def __init_subclass__(cls) -> None: # pragma: no cover
|
||
pass
|
||
|
||
def add(self, key: str, n: int = 1) -> None:
|
||
self.counts[key] = self.counts.get(key, 0) + n
|
||
|
||
def skip(self, key: str, n: int = 1) -> None:
|
||
self.skips[key] = self.skips.get(key, 0) + n
|
||
|
||
def note(self, message: str) -> None:
|
||
if message not in self.notes:
|
||
self.notes.append(message)
|
||
|
||
def dump(self, apply: bool) -> None:
|
||
mode = "已落库" if apply else "干跑(未写库)"
|
||
print(f"\n{'=' * 68}\n导入报告 {mode}\n{'=' * 68}")
|
||
for key in sorted(self.counts):
|
||
print(f" {key:<44s} {self.counts[key]:>8,}")
|
||
if self.skips:
|
||
print("\n 跳过:")
|
||
for key in sorted(self.skips):
|
||
print(f" {key:<42s} {self.skips[key]:>8,}")
|
||
if self.notes:
|
||
print("\n 需要你确认的:")
|
||
for item in self.notes:
|
||
print(f" - {item}")
|
||
print("=" * 68)
|
||
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 小工具
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def dec(value: object) -> Decimal:
|
||
if value is None or value == "":
|
||
return ZERO
|
||
if isinstance(value, Decimal):
|
||
return value
|
||
try:
|
||
return Decimal(str(value))
|
||
except (InvalidOperation, ValueError):
|
||
return ZERO
|
||
|
||
|
||
# 老库的比率列有两套口径,混用一次就是一次静默的 100 倍误差,所以拆成两个函数、
|
||
# 每个调用点各自认领,不再用一个"看着像就乘 100"的启发式。
|
||
#
|
||
# *_margin(gross_profit_margin / gross_profit_a_margin)—— 已经是百分数:
|
||
# 实测 id=7 contract=12990 gross_profit=1578.60 margin=12.15,
|
||
# 1578.60 / 12990 × 100 = 12.15,对得上。而且库里真有 0.03 / 0.04 / 0.20
|
||
# 这种小于 1 的利润率,那是 0.03%,不是 3%。
|
||
# tax_rate(project_estimate_*、project_budget_*)—— 是小数:
|
||
# 全库 min=0 max=0.13,0.13 显然是 13%。
|
||
#
|
||
# 之前一个 to_percent 用 -1 < v < 1 同时套两类列,后果是 6 条利润率被放大 100 倍。
|
||
|
||
# 比率绝对值超过这个数就当脏值报出来(老库 P00217 是 contract_amount=1.00
|
||
# 的试算项目,毛利 -495.04 除出 -49504%),列宽放宽后能存下,但得让人知道。
|
||
ABSURD_RATE = Decimal("10000")
|
||
|
||
|
||
def margin_rate(value: object, report: Report, column: str) -> Decimal:
|
||
"""老 ``*_margin`` 列:本身就是百分数,原样搬运,只做脏值体检。"""
|
||
raw = dec(value)
|
||
if raw > ABSURD_RATE or raw < -ABSURD_RATE:
|
||
report.skip(f"项目:{column} 超出 ±10000%,按原值导入(老数据脏值)")
|
||
report.note(f"老库 {column} 存在 ±10000% 以外的比率(如 project_no=P00217 "
|
||
"contract_amount=1.00 的试算项目),已按原值导入,"
|
||
"这类项目多为测试数据,建议核对后清理")
|
||
return raw
|
||
|
||
|
||
def tax_rate_percent(value: object, report: Report) -> Decimal:
|
||
"""老 ``tax_rate`` 列:小数口径(0.13)→ 百分数(13)。
|
||
|
||
只放大 (0, 1] 区间。税率不会是负数,所以敢用闭区间;万一有老行已经写成 13,
|
||
落在区间外原样保留,不会再乘一次。
|
||
"""
|
||
raw = dec(value)
|
||
if raw == ZERO:
|
||
return ZERO
|
||
if 0 < raw <= 1:
|
||
report.add("换算:税率小数→百分数")
|
||
return raw * Decimal("100")
|
||
return raw
|
||
|
||
|
||
def month_of(value: object) -> str:
|
||
"""datetime / 'YYYY-MM-DD' / 'YYYY-MM' 统一成 'YYYY-MM'。"""
|
||
if not value:
|
||
return ""
|
||
text_value = str(value).strip()
|
||
if len(text_value) >= 7 and text_value[4] == "-":
|
||
return text_value[:7]
|
||
return ""
|
||
|
||
|
||
def to_date(value: object) -> "date | None":
|
||
"""转成 date,口径与 app/api/projects._date 一致:只给到月就补当月 1 号。
|
||
|
||
新库 plan_start / plan_end / pay_date / receive_date 都是 DATE 列,空值一律给
|
||
NULL 而不是空串——MySQL 对 DATE 列写 '' 直接抛 1292 Incorrect date value。
|
||
date_of() 那种"截前 10 个字符、空就返回空串"的写法就是为这个坑生出来的,已删。
|
||
"""
|
||
if not value:
|
||
return None
|
||
text = str(value).strip()
|
||
if len(text) == 7:
|
||
text += "-01"
|
||
try:
|
||
return date.fromisoformat(text[:10])
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def is_deleted(value: object) -> bool:
|
||
"""老库的布尔列有三种长相,一种比一种坑。
|
||
|
||
* ``project.deleted`` 是 varchar:'0' / '1' / '' / NULL
|
||
* ``sys_role.enabled`` 之类是 tinyint:0 / 1
|
||
* ``sys_user.is_deleted`` 是 **bit(1)**,pymysql 取回来是 ``b'\\x00'`` / ``b'\\x01'``
|
||
|
||
第三种最容易翻车:``bool(b'\\x00')`` 是 **True**,``if row["is_deleted"]`` 会把
|
||
全部记录当成已删除——实测 125 个老账号因此一个都没导进来。
|
||
所以字节串单独判"有没有非零字节",别走真值判断。
|
||
"""
|
||
if value is None:
|
||
return False
|
||
if isinstance(value, (bytes, bytearray)):
|
||
return any(byte for byte in value)
|
||
if isinstance(value, bool):
|
||
return value
|
||
text_value = str(value).strip().lower()
|
||
return text_value in {"1", "true", "y", "yes"}
|
||
|
||
|
||
def procurement_type_names(legacy: Session) -> dict[str, str]:
|
||
"""老库 procurement_type 字典:id 字符串 -> 中文类别名。
|
||
|
||
project_budget_cost_detail.category 存的是这张表的 id("8"、"12"…),
|
||
不是中文名。之前拿 category 直接去查中文别名表,609 行采购明细一行都没归对
|
||
产品归属,项目主档的华智/华三/汇智/紫光其他四个金额列自然全是 0。
|
||
"""
|
||
try:
|
||
rows = fetch_all(legacy, "SELECT id, name FROM procurement_type")
|
||
except Exception: # noqa: BLE001 老库版本没有这张字典表时退回内置副本
|
||
return {str(key): value for key, value in LEGACY_PROCUREMENT_TYPES.items()}
|
||
mapping = {str(row["id"]): (row["name"] or "").strip() for row in rows}
|
||
return mapping or {str(k): v for k, v in LEGACY_PROCUREMENT_TYPES.items()}
|
||
|
||
|
||
def resolve_category(raw: object, dictionary: dict[str, str],
|
||
report: Report) -> tuple[str, str]:
|
||
"""老 category -> (中文类别名, 产品归属编码)。
|
||
|
||
返回的中文名写进 category 列(新系统这列就是给人看的),
|
||
编码写进 product_category(BUD-COST-002 按它汇总四个产品金额)。
|
||
字典里查不到又不是一个中文名(比如一个没见过的数字码),就原样保留类别名并计数。
|
||
"""
|
||
text_value = ("" if raw is None else str(raw)).strip()
|
||
if not text_value:
|
||
return "", ""
|
||
name = dictionary.get(text_value, text_value)
|
||
if text_value in dictionary:
|
||
report.add("归一化:采购类别 id → 中文名")
|
||
product_category = LEGACY_CATEGORY_ALIASES.get(name, "")
|
||
if name and not product_category:
|
||
UNMAPPED_CATEGORY[name] = UNMAPPED_CATEGORY.get(name, 0) + 1
|
||
return name[:128], product_category
|
||
|
||
|
||
def candidates_from_legacy(row: dict, role_map: dict[int, int], report: Report) -> list[dict]:
|
||
"""act_task_def 的三个 candidate_* 字符串 → candidates_json。
|
||
|
||
老库存的是逗号分隔的 id 串(`'15'`、`'34,35'`),空串和 NULL 混用。
|
||
"""
|
||
out: list[dict] = []
|
||
roles = [int(x) for x in str(row.get("candidate_roles") or "").replace(";", ",").split(",")
|
||
if x.strip().isdigit()]
|
||
mapped = [role_map[legacy_id] for legacy_id in roles if legacy_id in role_map]
|
||
if roles and not mapped:
|
||
report.skip("流程节点:候选人角色在角色映射里找不到")
|
||
elif mapped:
|
||
out.append({"type": "role", "ids": mapped})
|
||
|
||
users = [x.strip() for x in str(row.get("candidate_users") or "").replace(";", ",").split(",")
|
||
if x.strip()]
|
||
if users:
|
||
# 老库这里存的多是用户 id;对不上的原样记在 remark 里,不猜
|
||
report.note("act_task_def.candidate_users 是老库用户 id,新系统需按账号重新指定,"
|
||
"脚本只迁了角色候选人")
|
||
return out
|
||
|
||
|
||
def module_of_legacy_flow(row: dict) -> str:
|
||
key = str(row.get("proc_def_key") or "").lower()
|
||
name = str(row.get("proc_def_name") or "")
|
||
for token, module in (("estimate", "estimate"), ("budget", "budget"),
|
||
("settle", "settlement"), ("final", "final")):
|
||
if token in key:
|
||
return module
|
||
for token, module in (("概算", "estimate"), ("预算", "budget"),
|
||
("结算", "settlement"), ("决算", "final")):
|
||
if token in name:
|
||
return module
|
||
return ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 各步骤
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def fetch_all(legacy: Session, sql: str, params: dict | None = None) -> list[dict]:
|
||
return [dict(row) for row in legacy.execute(text(sql), params or {}).mappings().all()]
|
||
|
||
|
||
_TABLE_NAMES: dict[str, set[str]] = {}
|
||
|
||
|
||
def legacy_table_names(legacy: Session) -> set[str]:
|
||
"""老库里现有的表名,一次查询缓存住。
|
||
|
||
老库有几张 _temp 草稿表是残缺的(project_budget_cost_manage 就没有 _temp),
|
||
拼表名前得先问一句库里到底有没有,不能假设"正式表有、草稿表也一定有"。
|
||
"""
|
||
key = "tables"
|
||
if key not in _TABLE_NAMES:
|
||
_TABLE_NAMES[key] = {row[0] for row in legacy.execute(
|
||
text("SELECT TABLE_NAME FROM information_schema.TABLES "
|
||
"WHERE TABLE_SCHEMA = DATABASE()")).fetchall()}
|
||
return _TABLE_NAMES[key]
|
||
|
||
|
||
def make_table_fn(legacy: Session, suffix: str, report: Report):
|
||
"""返回 table(正式表名) → 实际要查的表名(正式表或 _temp 草稿表)。
|
||
|
||
解析结果按表名缓存,所以"草稿表不存在、退回正式表"这条只会在报告里出现一次,
|
||
不会被每个项目刷一遍。
|
||
"""
|
||
available = legacy_table_names(legacy)
|
||
resolved: dict[str, str] = {}
|
||
|
||
def table(name: str) -> str:
|
||
if name not in resolved:
|
||
target = name + suffix
|
||
if suffix and target not in available:
|
||
report.skip(f"老库没有 {target},该项改用正式表 {name}")
|
||
report.note(f"老库 {name} 没有对应的 _temp 草稿表,--with-temp 下该项仍读正式表")
|
||
target = name
|
||
resolved[name] = target
|
||
return resolved[name]
|
||
|
||
return table
|
||
|
||
|
||
def migrate_org(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
rows = fetch_all(legacy, "SELECT id, name, parent_id, level, manager_id, enabled FROM dept "
|
||
"ORDER BY parent_id IS NULL, level, id")
|
||
for row in rows:
|
||
unit = OrgUnit(
|
||
name=row["name"] or f"部门{row['id']}",
|
||
code=f"legacy_{row['id']}",
|
||
parent_id=maps["dept"].get(row["parent_id"]) if row["parent_id"] else None,
|
||
level=row["level"] or 1,
|
||
order_number=100,
|
||
status=Status.ACTIVE if (row["enabled"] in (None, 1)) else Status.DISABLED,
|
||
remark="由老四算 dept 导入",
|
||
)
|
||
db.add(unit)
|
||
db.flush()
|
||
maps["dept"][row["id"]] = unit.id
|
||
report.add("org_unit")
|
||
# path 建完再补,省得递归
|
||
for legacy_id, new_id in maps["dept"].items():
|
||
chain: list[int] = []
|
||
unit = db.get(OrgUnit, new_id)
|
||
while unit is not None and unit.id not in chain:
|
||
chain.append(unit.id)
|
||
unit = db.get(OrgUnit, unit.parent_id) if unit.parent_id else None
|
||
unit_row = db.get(OrgUnit, new_id)
|
||
if unit_row is not None:
|
||
unit_row.path = "/" + "/".join(str(item) for item in reversed(chain))
|
||
|
||
|
||
def migrate_positions(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
"""岗位按**名字**去重。
|
||
|
||
老库一批人 position_id = 0(没设岗位但填了岗位名),拿 id 拼 code 一定撞唯一键;
|
||
同一个岗位名也可能挂在不同 id 下,按名字建才不会造出一堆同名岗位。
|
||
"""
|
||
rows = fetch_all(legacy, "SELECT position_id, position_name FROM sys_user "
|
||
"WHERE position_name IS NOT NULL AND position_name <> '' "
|
||
"ORDER BY position_id")
|
||
by_name: dict[str, int] = {}
|
||
for row in rows:
|
||
name = row["position_name"].strip()
|
||
if not name:
|
||
continue
|
||
if name not in by_name:
|
||
position = Position(name=name, code=f"legacy_pos_{len(by_name) + 1}",
|
||
remark="由老四算 sys_user 导入")
|
||
db.add(position)
|
||
db.flush()
|
||
by_name[name] = position.id
|
||
report.add("org_position")
|
||
maps["position"][row["position_id"]] = by_name[name]
|
||
|
||
|
||
def migrate_users(legacy: Session, db: Session, report: Report, opts: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
rows = fetch_all(
|
||
legacy,
|
||
"SELECT id, user_name, real_name, emp_code, telephone, company_email, enabled, "
|
||
" dept_id, dept_name, position_id, position_name, direct_manager, work_location, "
|
||
" role_id, is_deleted, login_date, remark "
|
||
"FROM sys_user ORDER BY id")
|
||
by_name: dict[str, int] = {}
|
||
for row in rows:
|
||
if is_deleted(row["is_deleted"]):
|
||
report.skip("账号:老库已删除")
|
||
continue
|
||
account_key = (row["user_name"] or "").strip()
|
||
if not account_key:
|
||
report.skip("账号:老库 user_name 为空")
|
||
continue
|
||
exists = db.execute(select(Account.id).where(Account.account == account_key)).scalar()
|
||
if exists:
|
||
# 已存在(seed 的 admin 或导过一半)就复用,不重复插
|
||
maps["user"][row["id"]] = exists
|
||
report.skip("账号:账号名已存在,复用新库记录")
|
||
continue
|
||
display = (row["real_name"] or account_key).strip()
|
||
account = Account(
|
||
account=account_key,
|
||
# 老库口令是 sha+salt,算法不在本系统里,搬不过来也校验不了。
|
||
# 默认留空 = 不能本地登录(走统一身份);要能本地登录就显式给一个初始口令。
|
||
password_hash=hash_password(opts.password_for_migrated)
|
||
if opts.password_for_migrated else "",
|
||
display_name=display,
|
||
emp_code=row["emp_code"] or "",
|
||
telephone=row["telephone"] or "",
|
||
email=row["company_email"] or "",
|
||
work_location=row["work_location"] or "",
|
||
org_unit_id=maps["dept"].get(row["dept_id"]),
|
||
position_id=maps["position"].get(row["position_id"]),
|
||
status=Status.ACTIVE if row["enabled"] else Status.DISABLED,
|
||
source="local",
|
||
remark="由老四算 sys_user 导入",
|
||
)
|
||
db.add(account)
|
||
db.flush()
|
||
maps["user"][row["id"]] = account.id
|
||
by_name.setdefault(display, account.id)
|
||
report.add("accounts")
|
||
|
||
if account.org_unit_id:
|
||
db.add(AccountIdentity(account_id=account.id, org_unit_id=account.org_unit_id,
|
||
position_id=account.position_id, is_default=True))
|
||
report.add("account_identity")
|
||
|
||
# direct_manager 存的是姓名不是 id,只能按姓名回查;查不到就留空并计数
|
||
for row in rows:
|
||
name = (row["direct_manager"] or "").strip()
|
||
new_id = maps["user"].get(row["id"])
|
||
if not new_id or not name:
|
||
continue
|
||
target = by_name.get(name)
|
||
if target:
|
||
account = db.get(Account, new_id)
|
||
if account is not None:
|
||
account.leader_id = target
|
||
else:
|
||
report.skip("账号:直属上级姓名在新库里匹配不到")
|
||
|
||
# 老 sys_user.role_id 是单角色,映射成新库的 账号-角色 关系
|
||
for row in rows:
|
||
legacy_role = row["role_id"]
|
||
new_account = maps["user"].get(row["id"])
|
||
if not legacy_role or not new_account:
|
||
continue
|
||
new_role = maps["role"].get(legacy_role)
|
||
if not new_role:
|
||
report.skip("角色关系:老角色未导入")
|
||
continue
|
||
exists = db.execute(select(PermAccountRole).where(
|
||
PermAccountRole.account_id == new_account,
|
||
PermAccountRole.role_id == new_role)).scalar()
|
||
if exists:
|
||
continue
|
||
db.add(PermAccountRole(account_id=new_account, role_id=new_role))
|
||
report.add("perm_account_role")
|
||
|
||
|
||
def migrate_roles(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
rows = fetch_all(legacy, "SELECT id, name, level FROM sys_role "
|
||
"WHERE is_deleted = 0 OR is_deleted IS NULL ORDER BY id")
|
||
for row in rows:
|
||
name = (row["name"] or "").strip()
|
||
# 名字能对上现成角色就复用,别造一个 "财务人员" 又留一个 finance
|
||
existing = db.execute(select(PermRole.id).where(PermRole.name == name,
|
||
PermRole.is_deleted.is_(False))).scalar()
|
||
if existing:
|
||
maps["role"][row["id"]] = existing
|
||
report.skip("角色:同名角色已存在,复用")
|
||
continue
|
||
role = PermRole(name=name, code=f"legacy_{row['id']}",
|
||
remark=f"由老四算 sys_role 导入(level={row['level']})",
|
||
order_number=(row["level"] or 0) * 10)
|
||
db.add(role)
|
||
db.flush()
|
||
maps["role"][row["id"]] = role.id
|
||
report.add("perm_role")
|
||
|
||
|
||
def migrate_flows(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
rows = fetch_all(legacy, "SELECT proc_def_key, proc_def_name, task_key, task_name, task_type, "
|
||
"task_index, rollback_task_key, candidate_roles, candidate_users, "
|
||
"candidate_types FROM act_task_def "
|
||
"ORDER BY proc_def_key, task_index, id")
|
||
grouped: dict[str, list[dict]] = {}
|
||
for row in rows:
|
||
grouped.setdefault(str(row["proc_def_key"]), []).append(row)
|
||
|
||
for proc_key, nodes in grouped.items():
|
||
module = module_of_legacy_flow(nodes[0])
|
||
if not module:
|
||
report.skip(f"流程:{proc_key} 不属于本项目范围(概算/预算/结算/决算),未导入")
|
||
continue
|
||
existing = db.execute(select(ApprovalFlow.id).where(
|
||
ApprovalFlow.key == f"legacy_{proc_key}",
|
||
ApprovalFlow.is_deleted.is_(False))).scalar()
|
||
if existing:
|
||
maps["flow"][proc_key] = existing
|
||
report.skip("流程:同 key 已导入")
|
||
continue
|
||
flow = ApprovalFlow(
|
||
key=f"legacy_{proc_key}", name=nodes[0]["proc_def_name"] or proc_key,
|
||
business_module=module, version=1, status="published", priority=200,
|
||
auto_skip_duplicate=True,
|
||
remark="由老四算 act_task_def 导入,请复核候选人与兜底策略后再启用",
|
||
)
|
||
db.add(flow)
|
||
db.flush()
|
||
maps["flow"][proc_key] = flow.id
|
||
report.add("approval_flow")
|
||
|
||
seen_keys: set[str] = set()
|
||
for order, node in enumerate(nodes, start=1):
|
||
# 老 act_task_def 里同一个 proc_def 下会有重复 task_key(按部门条件重复配的同名节点),
|
||
# 新表有 (flow_id, node_key) 唯一约束,留第一条,其余计数跳过
|
||
node_key = node["task_key"] or f"n{order}"
|
||
if node_key in seen_keys:
|
||
report.skip("流程节点:同一流程里 node_key 重复,只保留第一条")
|
||
continue
|
||
seen_keys.add(node_key)
|
||
candidates = candidates_from_legacy(node, maps["role"], report)
|
||
rollback = [x.strip() for x in str(node.get("rollback_task_key") or "").split(",")
|
||
if x.strip()]
|
||
db.add(ApprovalNode(
|
||
flow_id=flow.id,
|
||
node_key=node_key,
|
||
name=node["task_name"] or f"节点{order}",
|
||
node_type=NodeType.APPROVE,
|
||
order_number=(node["task_index"] or order) * 10,
|
||
sign_type=LEGACY_SIGN_TYPES.get(node["task_type"], SignType.ANY),
|
||
candidates_json=json.dumps(candidates, ensure_ascii=False),
|
||
# 老系统找不到审批人时报空指针,这里一律显式兜底给管理员
|
||
empty_strategy="admin",
|
||
rollback_keys_json=json.dumps(rollback, ensure_ascii=False),
|
||
reject_to="end",
|
||
reject_comment_required=True,
|
||
enabled=True,
|
||
remark=("老 candidate_types=" + str(node.get("candidate_types"))
|
||
if node.get("candidate_types") else ""),
|
||
))
|
||
report.add("approval_node")
|
||
report.note("导入的流程候选人可能为空,请到「流程配置」里复核;"
|
||
"老流程的 task_type 全是或签,会签需求要手工调")
|
||
|
||
|
||
def fallback_account(db: Session) -> int | None:
|
||
"""creator_id 这类列是 NOT NULL 且带外键,映射不到时不能塞 0。
|
||
|
||
退给系统管理员,并在报告里计数——归属错了还能改,插不进去才是真麻烦。
|
||
"""
|
||
if "admin_account" not in _FALLBACK:
|
||
_FALLBACK["admin_account"] = db.execute(
|
||
select(Account.id).where(Account.is_system_admin.is_(True),
|
||
Account.is_deleted.is_(False)).limit(1)
|
||
).scalar()
|
||
return _FALLBACK["admin_account"]
|
||
|
||
|
||
def _new_revision(db: Session, project_id: int, module: str, status: str, title: str,
|
||
creator_id: int | None, creator_name: str) -> Revision:
|
||
revision = Revision(module=module, project_id=project_id, version=1, status=status,
|
||
title=title, creator_id=creator_id or fallback_account(db) or 0,
|
||
creator_name=creator_name)
|
||
db.add(revision)
|
||
db.flush()
|
||
return revision
|
||
|
||
|
||
def migrate_projects(legacy: Session, db: Session, report: Report, opts: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
limit = f" LIMIT {int(opts.limit_projects)}" if opts.limit_projects else ""
|
||
rows = fetch_all(
|
||
legacy,
|
||
"SELECT * FROM project "
|
||
"WHERE deleted IS NULL OR deleted = '' OR deleted IN ('0','false') "
|
||
f"ORDER BY id{limit}")
|
||
for row in rows:
|
||
legacy_id = row["id"]
|
||
project = Project(
|
||
project_no=row.get("project_no") or f"L{legacy_id}",
|
||
name=(row.get("name") or f"老项目{legacy_id}")[:255],
|
||
other_name=(row.get("other_name") or "")[:255],
|
||
type=row.get("type") or 1,
|
||
stage=row.get("stage") if row.get("stage") is not None else ProjectStage.S0_ESTIMATE,
|
||
stage_remark=row.get("stage_remark") or "",
|
||
next_plan=row.get("next_plan") or "",
|
||
creator_id=maps["user"].get(row.get("creator_id")) or fallback_account(db),
|
||
creator_name=row.get("creator_name") or "",
|
||
org_unit_id=maps["dept"].get(row.get("dept_id")),
|
||
org_unit_name=row.get("dept_name") or "",
|
||
principal=(row.get("principal") or "")[:100],
|
||
customer=(row.get("customer") or "")[:255],
|
||
terminal_customer=(row.get("terminal_customer") or "")[:255],
|
||
plan_start=to_date(row.get("plan_start_str")) or to_date(row.get("start_date")),
|
||
plan_end=to_date(row.get("plan_end_str")) or to_date(row.get("end_date")),
|
||
contract_time=row.get("contract_time"),
|
||
bids_time=row.get("bids_time"),
|
||
# 公司销售阶段与数据来源:source 这一列是清理脚本认"迁移数据"的唯一依据,
|
||
# 少了它,下次清演示数据就只能把这八百多个真实项目一起冲掉。
|
||
sale_stage=row.get("sale_stage") if row.get("sale_stage") is not None
|
||
else SaleStage.UNSET,
|
||
source=ProjectSource.LEGACY,
|
||
underwritten_mode=row.get("underwritten_mode") or 0,
|
||
cooperate_type=row.get("cooperate_type") or 0,
|
||
collaborator=(row.get("collaborator") or "")[:255],
|
||
certainty=row.get("certainty") or 0,
|
||
is_second=row.get("is_second") or 0,
|
||
sign_type=row.get("sign_type") or 0,
|
||
resolve_plan=row.get("resolve_plan") or 0,
|
||
industry_scene=row.get("industry_scene") or 0,
|
||
industry_scene_desc=(row.get("industry_scenario") or "")[:255],
|
||
main_contract_collection_terms=row.get("main_contract_collection_terms") or "",
|
||
main_contract_resolve_plan=row.get("main_contract_resolve_plan") or "",
|
||
calculation_collection=row.get("calculation_collection") or "",
|
||
value_risk=row.get("value_risk") or "",
|
||
huazhi_amount=dec(row.get("huazhi_product_amount")),
|
||
huizhi_amount=dec(row.get("huizhi_product_amount")),
|
||
h3c_amount=dec(row.get("huasan_product_amount")),
|
||
other_product_amount=dec(row.get("ziguang_other_amount")),
|
||
contract_amount=dec(row.get("contract_amount")),
|
||
gross_profit_a=dec(row.get("gross_profit_a")),
|
||
gross_profit_a_rate=margin_rate(row.get("gross_profit_a_margin"),
|
||
report, "gross_profit_a_margin"),
|
||
gross_profit=dec(row.get("gross_profit")),
|
||
gross_profit_rate=margin_rate(row.get("gross_profit_margin"),
|
||
report, "gross_profit_margin"),
|
||
advance_peak_amount=dec(row.get("advance_peak_amount")),
|
||
advance_interest_amount=dec(row.get("advance_interest_amount")),
|
||
# 老库只存垫资利息金额、不存利率,利率得外部给一个,否则资金计划算不出来
|
||
advance_interest_rate=Decimal(str(opts.advance_rate)),
|
||
calc_version="",
|
||
)
|
||
db.add(project)
|
||
db.flush()
|
||
maps["project"][legacy_id] = project.id
|
||
report.add("project")
|
||
if row.get("creator_id") and row["creator_id"] not in maps["user"]:
|
||
report.skip("项目:创建人在新库里映射不到,归属改挂到系统管理员")
|
||
if opts.advance_rate and maps["project"]:
|
||
report.note(f"老库没有垫资年利率字段,本次按 --advance-rate={opts.advance_rate}% 统一赋值;"
|
||
"各项目利率请在项目档案里逐个核对")
|
||
|
||
|
||
def migrate_estimate(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
incomes = fetch_all(legacy, "SELECT project_id, type, income_tax_include, tax_rate "
|
||
"FROM project_estimate_income")
|
||
costs = fetch_all(legacy, "SELECT project_id, type, cost_tax_include, tax_rate "
|
||
"FROM project_estimate_cost")
|
||
manages = fetch_all(legacy, "SELECT project_id, type, cost_tax_exclude "
|
||
"FROM project_estimate_cost_manage")
|
||
grouped: dict[int, dict[str, list]] = {}
|
||
for row in incomes:
|
||
grouped.setdefault(row["project_id"], {"income": [], "cost": [], "manage": []})["income"].append(row)
|
||
for row in costs:
|
||
grouped.setdefault(row["project_id"], {"income": [], "cost": [], "manage": []})["cost"].append(row)
|
||
for row in manages:
|
||
grouped.setdefault(row["project_id"], {"income": [], "cost": [], "manage": []})["manage"].append(row)
|
||
|
||
for legacy_project, buckets in grouped.items():
|
||
project_id = maps["project"].get(legacy_project)
|
||
if not project_id:
|
||
report.skip("概算明细:所属项目未导入")
|
||
continue
|
||
if not any(buckets.values()):
|
||
continue
|
||
revision = _new_revision(db, project_id, "estimate", RevisionStatus.DRAFT,
|
||
"由老四算导入", None, "")
|
||
maps["revision"][(legacy_project, "estimate")] = revision.id
|
||
for order, row in enumerate(buckets["income"], start=1):
|
||
db.add(EstimateIncomeLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
income_type=row["type"] or 1,
|
||
name=ESTIMATE_INCOME_NAMES.get(row["type"], f"收入项{order}"),
|
||
include_amount=dec(row["income_tax_include"]),
|
||
tax_rate=tax_rate_percent(row["tax_rate"], report), sort=order * 10))
|
||
report.add("estimate_income_line")
|
||
for order, row in enumerate(buckets["cost"], start=1):
|
||
db.add(EstimateCostLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
purchase_type=row["type"] or 1,
|
||
name=ESTIMATE_COST_NAMES.get(row["type"], f"采购项{order}"),
|
||
include_amount=dec(row["cost_tax_include"]),
|
||
tax_rate=tax_rate_percent(row["tax_rate"], report), sort=order * 10))
|
||
report.add("estimate_cost_line")
|
||
for order, row in enumerate(buckets["manage"], start=1):
|
||
db.add(EstimateOtherLine(
|
||
revision_id=revision.id, project_id=project_id, bucket="project_manage",
|
||
name=f"项目管理费用-{row['type'] or order}",
|
||
exclude_amount=dec(row["cost_tax_exclude"]), manual=True, sort=order * 10))
|
||
report.add("estimate_other_line")
|
||
report.note("老概算明细表没有明细名称列,名称按类型生成,必要时人工补")
|
||
|
||
|
||
def migrate_budget(legacy: Session, db: Session, report: Report, opts: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
suffix = "_temp" if opts.with_temp else ""
|
||
if opts.with_temp:
|
||
report.note("启用了 --with-temp:导入的是老库 _temp 草稿表,不是已提交的正式表")
|
||
table = make_table_fn(legacy, suffix, report)
|
||
categories = procurement_type_names(legacy)
|
||
|
||
projects_with_fund = 0
|
||
projects_without_fund = 0
|
||
|
||
projects_with_budget = fetch_all(
|
||
legacy, f"SELECT DISTINCT project_id FROM {table('project_budget_income_detail')} "
|
||
"UNION SELECT DISTINCT project_id FROM "
|
||
f"{table('project_budget_cost_detail')}")
|
||
|
||
for row in projects_with_budget:
|
||
project_id = maps["project"].get(row["project_id"])
|
||
if not project_id:
|
||
report.skip("预算明细:所属项目未导入")
|
||
continue
|
||
revision = _new_revision(db, project_id, "budget", RevisionStatus.DRAFT,
|
||
"由老四算导入", None, "")
|
||
maps["revision"][(row["project_id"], "budget")] = revision.id
|
||
|
||
for order, item in enumerate(fetch_all(
|
||
legacy, f"SELECT type, name, spec, param, unit, amount, price, tax_rate, "
|
||
f"total_tax_include FROM {table('project_budget_income_detail')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY id"), start=1):
|
||
db.add(BudgetIncomeLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
income_type=item["type"] or 1, name=item["name"] or "",
|
||
spec=item["spec"] or "", param=item["param"] or "", unit=item["unit"] or "",
|
||
qty=dec(item["amount"]), unit_price=dec(item["price"]),
|
||
include_amount=dec(item["total_tax_include"]),
|
||
tax_rate=tax_rate_percent(item["tax_rate"], report), sort=order * 10))
|
||
report.add("budget_income_line")
|
||
|
||
for order, item in enumerate(fetch_all(
|
||
legacy, f"SELECT type, category, name, unit, amount, price, tax_rate, "
|
||
f"contract_party, is_underwritten, underwritten_amount, pay_time, "
|
||
f"pay_amount, pay_way, remark, total_tax_include "
|
||
f"FROM {table('project_budget_cost_detail')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY id"), start=1):
|
||
category, product_category = resolve_category(item["category"], categories, report)
|
||
db.add(BudgetPurchaseLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
purchase_type=item["type"] or 1, category=category,
|
||
product_category=product_category, name=item["name"] or "",
|
||
unit=item["unit"] or "", qty=dec(item["amount"]),
|
||
unit_price=dec(item["price"]),
|
||
include_amount=dec(item["total_tax_include"]),
|
||
tax_rate=tax_rate_percent(item["tax_rate"], report),
|
||
contract_party=item["contract_party"] or "",
|
||
is_underwritten=bool(item["is_underwritten"]),
|
||
underwritten_amount=dec(item["underwritten_amount"]),
|
||
pay_date=to_date(item["pay_time"]), pay_amount=dec(item["pay_amount"]),
|
||
pay_way=item["pay_way"] or "", remark=item["remark"] or "", sort=order * 10))
|
||
report.add("budget_purchase_line")
|
||
|
||
for order, item in enumerate(fetch_all(
|
||
legacy, f"SELECT type, cost_tax_exclude FROM {table('project_budget_cost_manage')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY id"), start=1):
|
||
db.add(BudgetManageLine(
|
||
revision_id=revision.id, project_id=project_id, manage_type=item["type"] or 0,
|
||
item_code=f"legacy_{item['type'] or order}",
|
||
item_name=f"项目管理费用-{item['type'] or order}",
|
||
total=dec(item["cost_tax_exclude"]), deletable=True,
|
||
counts_to_travel_limit=False, sort=order * 10))
|
||
report.add("budget_manage_line")
|
||
report.note("老项目管理费用只有类型与不含税金额,item_code / 是否计入差旅限额"
|
||
"需人工指定,否则 BUD-VAL-001 校验算不准")
|
||
|
||
for order, item in enumerate(fetch_all(
|
||
legacy, f"SELECT pay_point, pay_time, pay_project, pay_amount, supplier "
|
||
f"FROM {table('project_budget_pay_plan')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY id"), start=1):
|
||
db.add(BudgetPayPlanLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
pay_point=item["pay_point"] or "", pay_date=to_date(item["pay_time"]),
|
||
purchase_type=0, amount=dec(item["pay_amount"]), supplier=item["supplier"] or "",
|
||
remark=item["pay_project"] or "", sort=order * 10))
|
||
report.add("budget_pay_plan_line")
|
||
|
||
for order, item in enumerate(fetch_all(
|
||
legacy, f"SELECT receive_point, receive_time, receive_amount "
|
||
f"FROM {table('project_budget_income_plan_detail')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY id"), start=1):
|
||
db.add(BudgetIncomePlanLine(
|
||
revision_id=revision.id, project_id=project_id,
|
||
receive_point=item["receive_point"] or "",
|
||
receive_date=to_date(item["receive_time"]),
|
||
amount=dec(item["receive_amount"]), sort=order * 10))
|
||
report.add("budget_income_plan_line")
|
||
|
||
# 资金计划(按月):新系统有 budget_fund_month,老库对应 project_budget_plan_detail。
|
||
# 列名对不上但语义对得上,逐列显式搬:
|
||
# engineer_cost → construction_cost(施工支出)
|
||
# project_manage_cost → manage_cost(项目管理支出)
|
||
# earnest_money_cost / _income → earnest_cost / earnest_income(保证金)
|
||
# 老库没有"服务支出""其他支出""当月垫资年利率""期初余额"四列,留 0。
|
||
# total_cost / total_income / balance / underwritten_plan / repayment_plan /
|
||
# capital_interest 是派生列(模型注释:禁止其它代码改),这里一律不写,
|
||
# 交给项目里那一次「重新试算」按新口径算出来。
|
||
fund_rows = fetch_all(
|
||
legacy, f"SELECT month, device_cost, engineer_cost, project_manage_cost, "
|
||
f"earnest_money_cost, sale_income, earnest_money_income "
|
||
f"FROM {table('project_budget_plan_detail')} "
|
||
f"WHERE project_id = {row['project_id']} ORDER BY month, id")
|
||
seen_months: set[str] = set()
|
||
for order, item in enumerate(fund_rows, start=1):
|
||
month = month_of(item["month"])
|
||
if not month:
|
||
report.skip("资金计划:老库 month 不是 YYYY-MM,未导入")
|
||
continue
|
||
if month in seen_months:
|
||
# (revision_id, month) 是唯一键,老库同项目同月排重了就只能留一条
|
||
report.skip("资金计划:同项目同月有多行,只保留第一条")
|
||
continue
|
||
seen_months.add(month)
|
||
db.add(BudgetFundMonth(
|
||
revision_id=revision.id, project_id=project_id, month=month,
|
||
device_cost=dec(item["device_cost"]),
|
||
construction_cost=dec(item["engineer_cost"]),
|
||
service_cost=ZERO, other_cost=ZERO,
|
||
manage_cost=dec(item["project_manage_cost"]),
|
||
earnest_cost=dec(item["earnest_money_cost"]),
|
||
sale_income=dec(item["sale_income"]),
|
||
earnest_income=dec(item["earnest_money_income"]),
|
||
# 老库月计划是人工排的,不是从收付款计划汇总来的 → manual_override
|
||
manual_override=True,
|
||
# interest_rate / opening_balance 老库没有,留 0 由引擎按项目上的
|
||
# advance_interest_rate 重算
|
||
))
|
||
report.add("budget_fund_month")
|
||
if not fund_rows:
|
||
projects_without_fund += 1
|
||
else:
|
||
projects_with_fund += 1
|
||
|
||
if projects_without_fund:
|
||
report.note(f"{projects_without_fund} 个项目在老库没有月度资金计划(有预算明细但"
|
||
f" project_budget_plan_detail 无行),新系统里资金计划页会是空的,"
|
||
f"重新试算时按收付款计划推;另有 {projects_with_fund} 个项目带进了月度计划")
|
||
|
||
|
||
#: 老 project_visible.type 的真实语义(实测出来的,别信字段注释)
|
||
#:
|
||
#: 2 = 按**人员**授权可见,tid 是 sys_user.id
|
||
#: 1 = 按**角色**授权可见,tid 是 sys_role.id
|
||
#:
|
||
#: 实测依据(unis_fourcal_dev):全表 15,302 行都是 type=2,tid 有 35 个不同值、
|
||
#: 全部落在 sys_user.id 的 1~125 区间;其中 tid>33 的有 14,357 行,而 sys_role
|
||
#: 最大 id 才 33——那部分绝不可能是角色。tid 能撞上 sys_role 的只有 945 行
|
||
#: (全是 tid=1「四算系统管理员」,撞号而已)。
|
||
#:
|
||
#: 早先的版本把 type=2 当成角色,后果是两头都错:945 行按撞号插成了
|
||
#: project_visible_role(给"四算系统管理员"这个角色批了 833 个项目的可见性),
|
||
#: 另外 12,686 行真正的按人授权直接当"老角色未导入"丢掉。
|
||
LEGACY_VISIBLE_MEMBER = 2
|
||
LEGACY_VISIBLE_ROLE = 1
|
||
|
||
|
||
def visible_target(value: object) -> str:
|
||
"""老 project_visible.type -> 落到哪张新表;认不出来回空串。
|
||
|
||
只认整数和整数字符串(老库这列是 int,取回来可能是 int / Decimal / str),
|
||
带小数的值一律不猜——它是脏数据,报错比按 2 处理强。
|
||
"""
|
||
if isinstance(value, bool) or value is None:
|
||
return ""
|
||
if isinstance(value, float) and not value.is_integer():
|
||
return ""
|
||
try:
|
||
kind = int(value)
|
||
except (TypeError, ValueError):
|
||
return ""
|
||
if kind == LEGACY_VISIBLE_MEMBER:
|
||
return "member"
|
||
if kind == LEGACY_VISIBLE_ROLE:
|
||
return "role"
|
||
return ""
|
||
|
||
|
||
def migrate_visible(legacy: Session, db: Session, report: Report, _: argparse.Namespace,
|
||
maps: dict) -> None:
|
||
rows = fetch_all(legacy, "SELECT project_id, type, tid FROM project_visible")
|
||
# 逐行 SELECT 判重会在这张表上打 15,302 次查询,先把已有的两把唯一键捞进内存
|
||
seen_member = {(row.project_id, row.account_id)
|
||
for row in db.execute(select(ProjectMember)).scalars()}
|
||
seen_role = {(row.project_id, row.role_id)
|
||
for row in db.execute(select(ProjectVisibleRole)).scalars()}
|
||
# account_name 是冗余列,一次性把要用的账号名捞出来,别一人查一次
|
||
names: dict[int, str] = {}
|
||
account_ids = set(maps["user"].values())
|
||
if account_ids:
|
||
for account in db.execute(select(Account).where(Account.id.in_(account_ids))).scalars():
|
||
names[account.id] = account.display_name or account.account
|
||
|
||
for row in rows:
|
||
project_id = maps["project"].get(row["project_id"])
|
||
if not project_id:
|
||
report.skip("可见性:所属项目未导入")
|
||
continue
|
||
target = visible_target(row["type"])
|
||
if target == "member":
|
||
account_id = maps["user"].get(row["tid"])
|
||
if not account_id:
|
||
report.skip("可见性:授权人员未导入(老库已删除或缺账号名)")
|
||
continue
|
||
key = (project_id, account_id)
|
||
if key in seen_member:
|
||
continue
|
||
seen_member.add(key)
|
||
db.add(ProjectMember(project_id=project_id, account_id=account_id,
|
||
account_name=names.get(account_id, ""),
|
||
member_role=ProjectMemberRole.VIEWER))
|
||
report.add("project_member")
|
||
elif target == "role":
|
||
role_id = maps["role"].get(row["tid"])
|
||
if not role_id:
|
||
report.skip("可见性:老角色未导入")
|
||
continue
|
||
key = (project_id, role_id)
|
||
if key in seen_role:
|
||
continue
|
||
seen_role.add(key)
|
||
db.add(ProjectVisibleRole(project_id=project_id, role_id=role_id))
|
||
report.add("project_visible_role")
|
||
else:
|
||
report.skip(f"可见性:未预期的 type={row['type']}")
|
||
report.note("老 project_visible 的 type=2 是按人员授权(tid=sys_user.id),已导入 "
|
||
"project_member,member_role=viewer(仅可见);新系统里没有"
|
||
"\"整个角色对单个项目开放\"的界面入口,type=1 那类授权落到 project_visible_role")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 入口
|
||
# ---------------------------------------------------------------------------
|
||
|
||
STEPS: list[tuple[str, str, object]] = [
|
||
("org", "部门 → org_unit", migrate_org),
|
||
("role", "角色 → perm_role", migrate_roles),
|
||
("positions", "岗位 → org_position", migrate_positions),
|
||
("user", "用户 → accounts", migrate_users),
|
||
("flow", "act_task_def → 流程与节点", migrate_flows),
|
||
("project", "project → 项目主档", migrate_projects),
|
||
("estimate", "概算明细 → 概算行", migrate_estimate),
|
||
("budget", "预算明细 → 预算行", migrate_budget),
|
||
("visible", "project_visible → 项目可见角色", migrate_visible),
|
||
]
|
||
|
||
|
||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description="老四算 → 四算 PMS 数据导入(老库只读,默认干跑)")
|
||
parser.add_argument("--apply", action="store_true",
|
||
help="真正写库;不加这个参数只统计不落库")
|
||
parser.add_argument("--steps", default=",".join(key for key, _, _ in STEPS),
|
||
help=f"逗号分隔的步骤,默认全部:{','.join(key for key, _, _ in STEPS)}")
|
||
parser.add_argument("--limit-projects", type=int, default=0,
|
||
help="只导前 N 个项目(试水用),0 = 不限")
|
||
parser.add_argument("--advance-rate", default="6",
|
||
help="老库没有垫资年利率,这里给一个统一初始值(百分数)")
|
||
parser.add_argument("--password-for-migrated", default="",
|
||
help="给导入账号设一个统一初始口令;留空则账号不能本地登录")
|
||
parser.add_argument("--with-temp", action="store_true",
|
||
help="预算明细导 _temp 草稿表而不是已提交正式表")
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
opts = parse_args(argv)
|
||
chosen = [item for item in STEPS if item[0] in {s.strip() for s in opts.steps.split(",")}]
|
||
# 步骤之间有依赖,按 STEPS 的定义顺序跑,不受 --steps 书写顺序影响
|
||
print(f"档位={settings.env} 目标库={settings.database.name} "
|
||
f"老库={settings.legacy.name}(只读)")
|
||
if not settings.legacy.enabled:
|
||
print("配置里 [legacy].enabled = false,没有老库可读。", file=sys.stderr)
|
||
return 2
|
||
print("执行顺序:" + " → ".join(key for key, _, _ in chosen))
|
||
if not opts.apply:
|
||
print("当前是干跑,不会写任何数据。确认无误后加 --apply。")
|
||
|
||
engine = legacy_engine()
|
||
if engine is None:
|
||
print("拿不到老库连接(检查 [legacy] 配置)。", file=sys.stderr)
|
||
return 2
|
||
|
||
install_clip_guard()
|
||
|
||
report = Report()
|
||
maps: dict = {"dept": {}, "user": {}, "role": {}, "position": {}, "flow": {},
|
||
"project": {}, "revision": {}}
|
||
|
||
legacy = Session(engine)
|
||
try:
|
||
# 老库这一侧开只读事务,脚本里任何误写都会被 MySQL 直接拒绝
|
||
legacy.execute(text("START TRANSACTION READ ONLY"))
|
||
db = SessionFactory()()
|
||
try:
|
||
for _, label, func in chosen:
|
||
before = sum(report.counts.values())
|
||
func(legacy, db, report, opts, maps)
|
||
print(f" · {label:34s} 新增 {sum(report.counts.values()) - before:,}")
|
||
if opts.apply:
|
||
db.commit()
|
||
else:
|
||
db.rollback()
|
||
print("\n干跑结束,已全部回滚。")
|
||
except Exception:
|
||
db.rollback()
|
||
raise
|
||
finally:
|
||
db.close()
|
||
finally:
|
||
legacy.close()
|
||
|
||
for key, count in sorted(CLIPPED.items()):
|
||
report.skip(f"截断:{key} 超列宽 {count} 行")
|
||
if UNMAPPED_CATEGORY:
|
||
total = sum(UNMAPPED_CATEGORY.values())
|
||
top = sorted(UNMAPPED_CATEGORY.items(), key=lambda kv: (-kv[1], kv[0]))[:12]
|
||
report.skip(f"预算采购:类别不属于四个产品归属,product_category 留空"
|
||
f"({len(UNMAPPED_CATEGORY)} 种类别)", total)
|
||
report.note(
|
||
f"{total} 行采购明细的类别不在华智/汇智/华三/紫光其他四个产品归属内,"
|
||
f"product_category 只能留空,BUD-COST-002 汇总的四个产品金额因此少这部分。"
|
||
f"共 {len(UNMAPPED_CATEGORY)} 种类别,出现最多的:"
|
||
+ "、".join(f"{name}×{cnt}" for name, cnt in top)
|
||
)
|
||
if CLIPPED:
|
||
report.note("有字符串超过新库列宽,已按列宽截断(计数见上)。要么把模型里的列加宽到"
|
||
"够装老数据,要么确认这段文本可以丢;不要放着不管")
|
||
report.dump(opts.apply)
|
||
if opts.apply:
|
||
print("提示:导入后到项目里点一次「重新试算」,让计算快照和投影按新口径重算一遍。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|