129 lines
5.0 KiB
Python
129 lines
5.0 KiB
Python
"""后端启动期的 SQLite 等锁逻辑(``db.wait_for_lock``)。
|
||
|
||
线上现象:改了个文件、后端热重载之后就再也连不上了——浏览器里所有请求都在转圈,
|
||
``/health`` 也不返回。日志里其实是 ``sqlite3.OperationalError: database is locked``
|
||
+ ``Application startup failed. Exiting.``:上一个 worker 还握着写事务(内联生成任务的
|
||
会话要跨过几分钟的 ComfyUI 调用),新 worker 的 ``init_db()`` 抢不到锁直接退出,
|
||
而 ``--reload`` 的父进程仍然占着端口——连得上、永不响应,看起来就是「整个平台卡住」。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from sqlalchemy.exc import OperationalError
|
||
|
||
from app import db as dbmod
|
||
|
||
|
||
def _locked_error() -> OperationalError:
|
||
return OperationalError("UPDATE model_specs SET model_code='' ", {},
|
||
sqlite3.OperationalError("database is locked"))
|
||
|
||
|
||
def test_lock_error_is_recognised_but_real_sql_mistakes_are_not():
|
||
assert dbmod.is_lock_error(_locked_error()) is True
|
||
assert dbmod.is_lock_error(sqlite3.OperationalError("no such column: nope")) is False
|
||
assert dbmod.is_lock_error(RuntimeError("boom")) is False
|
||
|
||
|
||
def test_startup_maintenance_waits_out_a_transient_lock():
|
||
"""锁松开就立刻继续:热重载不再需要人工重启第二次。"""
|
||
calls = {"n": 0}
|
||
|
||
def run() -> str:
|
||
calls["n"] += 1
|
||
if calls["n"] < 3:
|
||
raise _locked_error()
|
||
return "done"
|
||
|
||
assert dbmod.wait_for_lock("回填新增列默认值", run, budget=10, interval=0.05) == "done"
|
||
assert calls["n"] == 3
|
||
|
||
|
||
def test_startup_maintenance_still_raises_real_errors_immediately():
|
||
"""语句本身写错时不许被「等锁」吞掉——否则一个 bug 会变成 60s 的静默停顿。"""
|
||
calls = {"n": 0}
|
||
|
||
def run() -> None:
|
||
calls["n"] += 1
|
||
raise OperationalError("SELECT 1", {}, sqlite3.OperationalError("no such table: model_specs"))
|
||
|
||
with pytest.raises(OperationalError):
|
||
dbmod.wait_for_lock("补齐新增列", run, budget=5, interval=0.05)
|
||
assert calls["n"] == 1
|
||
|
||
|
||
def test_giving_up_explains_how_to_fix_it():
|
||
with pytest.raises(RuntimeError) as caught:
|
||
dbmod.wait_for_lock("回填新增列默认值", lambda: (_ for _ in ()).throw(_locked_error()),
|
||
budget=0.2, interval=0.05)
|
||
message = str(caught.value)
|
||
assert "上一个后端进程" in message and "busy_timeout" in message and "回填新增列默认值" in message
|
||
|
||
|
||
def test_real_sqlite_write_lock_is_waited_for(tmp_path: Path):
|
||
"""真文件锁:另一个连接握着写事务时等它提交,然后维护语句正常执行。"""
|
||
path = tmp_path / "locked.db"
|
||
holder = sqlite3.connect(path, timeout=5, check_same_thread=False) # 由后台线程负责释放
|
||
holder.execute("CREATE TABLE t (x)")
|
||
holder.execute("INSERT INTO t VALUES (1)")
|
||
holder.commit()
|
||
holder.execute("BEGIN IMMEDIATE") # 独占写锁,直到 commit
|
||
|
||
released = {"at": 0.0}
|
||
|
||
def maintain() -> None:
|
||
conn = sqlite3.connect(path, timeout=0.05) # 几乎不等,把重试交给 wait_for_lock
|
||
try:
|
||
conn.execute("UPDATE t SET x = 2") # 需要写锁
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
released["at"] = time.monotonic()
|
||
|
||
def release_later() -> None:
|
||
time.sleep(0.6)
|
||
holder.commit()
|
||
holder.close()
|
||
|
||
thread = threading.Thread(target=release_later, daemon=True)
|
||
thread.start()
|
||
started = time.monotonic()
|
||
try:
|
||
dbmod.wait_for_lock("真锁验收", maintain, budget=15, interval=0.1)
|
||
finally:
|
||
thread.join(timeout=5)
|
||
assert released["at"] - started >= 0.4 # 确实等过了,不是碰运气一次就过
|
||
check = sqlite3.connect(path)
|
||
try:
|
||
assert check.execute("SELECT x FROM t").fetchone()[0] == 2
|
||
finally:
|
||
check.close()
|
||
|
||
|
||
def test_init_db_survives_a_busy_database(monkeypatch):
|
||
"""把 init_db 的四步全换成「先抛两次锁」,确认启动路径整体是等锁而不是崩掉。"""
|
||
counters = {"create_all": 0, "columns": 0, "relax": 0, "backfill": 0}
|
||
|
||
def flaky(key: str):
|
||
def run(*_args, **_kwargs) -> None:
|
||
counters[key] += 1
|
||
if counters[key] < 3:
|
||
raise _locked_error()
|
||
return run
|
||
|
||
metadata = type("M", (), {"create_all": staticmethod(flaky("create_all"))})()
|
||
monkeypatch.setattr(dbmod, "Base", type("B", (), {"metadata": metadata})())
|
||
monkeypatch.setattr(dbmod, "_add_missing_columns", flaky("columns"))
|
||
# tasks.project_id 放开非空要整表重建 tasks,这一步同样在启动路径上,必须一起等锁
|
||
monkeypatch.setattr(dbmod, "_relax_nullability", flaky("relax"))
|
||
monkeypatch.setattr(dbmod, "_backfill_defaults", flaky("backfill"))
|
||
monkeypatch.setattr(dbmod, "STARTUP_LOCK_BUDGET_SECONDS", 10.0)
|
||
|
||
dbmod.init_db()
|
||
assert counters == {"create_all": 3, "columns": 3, "relax": 3, "backfill": 3}
|