nex_animation/backend/tests/test_startup_lock.py

127 lines
4.8 KiB
Python

"""后端启动期的 SQLite 等锁逻辑(``db.wait_for_lock``)。
线上现象:改了个文件、后端热重载之后就再也连不上了——浏览器里所有请求都在转圈,
``/health`` 也不返回。日志里其实是 ``sqlite3.OperationalError: database is locked``
+ ``Application startup failed. Exiting.``:上一个 worker 还握着写事务(内联生成任务的
会话要跨过几分钟的 ComfyUI 调用),新 worker 的 ``init_db()`` 抢不到锁直接退出,
而 ``--reload`` 的父进程仍然占着端口——连得上、永不响应,看起来就是「整个平台卡住」。
"""
from __future__ import annotations
import sqlite3
import threading
import time
from pathlib import Path
import pytest
from sqlalchemy.exc import OperationalError
from app import db as dbmod
def _locked_error() -> OperationalError:
return OperationalError("UPDATE model_specs SET model_code='' ", {},
sqlite3.OperationalError("database is locked"))
def test_lock_error_is_recognised_but_real_sql_mistakes_are_not():
assert dbmod.is_lock_error(_locked_error()) is True
assert dbmod.is_lock_error(sqlite3.OperationalError("no such column: nope")) is False
assert dbmod.is_lock_error(RuntimeError("boom")) is False
def test_startup_maintenance_waits_out_a_transient_lock():
"""锁松开就立刻继续:热重载不再需要人工重启第二次。"""
calls = {"n": 0}
def run() -> str:
calls["n"] += 1
if calls["n"] < 3:
raise _locked_error()
return "done"
assert dbmod.wait_for_lock("回填新增列默认值", run, budget=10, interval=0.05) == "done"
assert calls["n"] == 3
def test_startup_maintenance_still_raises_real_errors_immediately():
"""语句本身写错时不许被「等锁」吞掉——否则一个 bug 会变成 60s 的静默停顿。"""
calls = {"n": 0}
def run() -> None:
calls["n"] += 1
raise OperationalError("SELECT 1", {}, sqlite3.OperationalError("no such table: model_specs"))
with pytest.raises(OperationalError):
dbmod.wait_for_lock("补齐新增列", run, budget=5, interval=0.05)
assert calls["n"] == 1
def test_giving_up_explains_how_to_fix_it():
with pytest.raises(RuntimeError) as caught:
dbmod.wait_for_lock("回填新增列默认值", lambda: (_ for _ in ()).throw(_locked_error()),
budget=0.2, interval=0.05)
message = str(caught.value)
assert "上一个后端进程" in message and "busy_timeout" in message and "回填新增列默认值" in message
def test_real_sqlite_write_lock_is_waited_for(tmp_path: Path):
"""真文件锁:另一个连接握着写事务时等它提交,然后维护语句正常执行。"""
path = tmp_path / "locked.db"
holder = sqlite3.connect(path, timeout=5, check_same_thread=False) # 由后台线程负责释放
holder.execute("CREATE TABLE t (x)")
holder.execute("INSERT INTO t VALUES (1)")
holder.commit()
holder.execute("BEGIN IMMEDIATE") # 独占写锁,直到 commit
released = {"at": 0.0}
def maintain() -> None:
conn = sqlite3.connect(path, timeout=0.05) # 几乎不等,把重试交给 wait_for_lock
try:
conn.execute("UPDATE t SET x = 2") # 需要写锁
conn.commit()
finally:
conn.close()
released["at"] = time.monotonic()
def release_later() -> None:
time.sleep(0.6)
holder.commit()
holder.close()
thread = threading.Thread(target=release_later, daemon=True)
thread.start()
started = time.monotonic()
try:
dbmod.wait_for_lock("真锁验收", maintain, budget=15, interval=0.1)
finally:
thread.join(timeout=5)
assert released["at"] - started >= 0.4 # 确实等过了,不是碰运气一次就过
check = sqlite3.connect(path)
try:
assert check.execute("SELECT x FROM t").fetchone()[0] == 2
finally:
check.close()
def test_init_db_survives_a_busy_database(monkeypatch):
"""把 init_db 的三步全换成「先抛两次锁」,确认启动路径整体是等锁而不是崩掉。"""
counters = {"create_all": 0, "columns": 0, "backfill": 0}
def flaky(key: str):
def run(*_args, **_kwargs) -> None:
counters[key] += 1
if counters[key] < 3:
raise _locked_error()
return run
metadata = type("M", (), {"create_all": staticmethod(flaky("create_all"))})()
monkeypatch.setattr(dbmod, "Base", type("B", (), {"metadata": metadata})())
monkeypatch.setattr(dbmod, "_add_missing_columns", flaky("columns"))
monkeypatch.setattr(dbmod, "_backfill_defaults", flaky("backfill"))
monkeypatch.setattr(dbmod, "STARTUP_LOCK_BUDGET_SECONDS", 10.0)
dbmod.init_db()
assert counters == {"create_all": 3, "columns": 3, "backfill": 3}