nex_animation/backend/tests/test_direct_connection.py

197 lines
9.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""ComfyUI 流量必须**直连**:环境里的系统代理不该替 ComfyUI 回答问题。
线上表现(本轮排查到的真实诱因):后端从配了系统代理的终端启动,shell 里的
``HTTP_PROXY`` 被 uvicorn 继承,于是连本机/局域网 ComfyUI 也要绕进本地代理
(Clash、Verge 之类)。后果有两层,都很坑:
* 端口已经关掉时,代理不会拒绝连接,而是替它回答一个**空 body 的 HTTP 502**。
502 在轮询容错里属于「服务抖一下」,于是「进程根本没起来」被误判成临时故障,
一路重试到 timeout,排查建议还指向错误的方向;
* 35GB 权重的上传/下载多走一跳本地代理,吞吐和超时都被别人牵着。
所以 ``comfy_common.client()`` 默认 ``trust_env=False``,并留下
``use_system_proxy`` / ``NEX_COMFY_USE_PROXY`` 作为显式开关。
"""
from __future__ import annotations
import re
import threading
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from app.providers import comfy_assets, comfy_common, comfyui
from app.providers.base import ProviderError, VideoGenRequest
from app.providers.comfy_common import client as comfy_client
from app.providers.comfy_assets import ComfyImageProvider, ComfyTTSProvider
from app.providers.comfyui import ComfyVideoProvider
from test_provider_outage import _free_base, fake_comfy, invalidate_reachability
# NO_PROXY 也要清:跑测试的 shell 常常自带 ``NO_PROXY="*"``,那样「代理伪装成 502」
# 这条链路根本不会被走到,反证用例就成了假绿。
PROXY_ENV = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy",
"NO_PROXY", "no_proxy")
@contextmanager
def fake_proxy(*, status: int = 502):
"""一个「什么都能答、答的却是网关错误」的本地代理,等价于 Clash 面对死端口。"""
state = {"hits": 0}
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *_args):
pass
def _answer(self):
state["hits"] += 1 # 记的是「这次请求有没有被代理截走」
self.send_response(status)
self.send_header("Content-Length", "0")
self.end_headers()
do_GET = do_POST = do_HEAD = _answer
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}", state
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
@pytest.fixture
def shell_proxy(monkeypatch):
"""把 shell 环境里的系统代理搬进用例(默认指向一个会答 502 的假代理)。"""
for key in PROXY_ENV:
monkeypatch.delenv(key, raising=False)
monkeypatch.delenv("NEX_COMFY_USE_PROXY", raising=False)
@contextmanager
def _start(**kw):
with fake_proxy(**kw) as (url, state):
for key in PROXY_ENV:
monkeypatch.setenv(key, url)
invalidate_reachability()
try:
yield url, state
finally:
invalidate_reachability()
return _start
# ---------------------------------------------------------------- 策略本身
def test_comfy_clients_ignore_the_inherited_shell_proxy_by_default(shell_proxy):
with shell_proxy():
assert comfy_common.use_system_proxy({}) is False
assert comfy_common.use_system_proxy(None) is False
with comfy_client(timeout=3) as http:
assert http._trust_env is False # noqa: SLF001 - 就是要钉住这个行为
def test_system_proxy_can_be_switched_back_on_explicitly(shell_proxy, monkeypatch):
with shell_proxy():
assert comfy_common.use_system_proxy({"use_system_proxy": True}) is True
monkeypatch.setenv("NEX_COMFY_USE_PROXY", "1")
assert comfy_common.use_system_proxy({}) is True
assert comfy_common.use_system_proxy(None) is True
assert comfy_common.use_system_proxy({"use_system_proxy": False}) is False # 配置优先
with comfy_client(timeout=3, config={"use_system_proxy": True}) as http:
assert http._trust_env is True # noqa: SLF001
def test_comfy_adapters_route_every_request_through_the_shared_client():
"""两个适配器里不该再出现裸 ``httpx.Client(``/``httpx.get(``——新代码也一样。"""
for module in (comfyui, comfy_assets, comfy_common):
assert getattr(module, "comfy_client", comfy_common.client) is comfy_common.client
for name in ("comfyui.py", "comfy_assets.py"):
source = Path(__file__).resolve().parent.parent.joinpath("app/providers", name).read_text()
leftovers = re.findall(r"httpx\.(?:get|post|put|Client)\(", source)
assert not leftovers, f"{name} 里还有绕过直连工厂的请求:{leftovers}"
# ---------------------------------------------------------------- 行为差异
def test_probe_reaches_the_real_comfyui_instead_of_the_proxy(shell_proxy):
with fake_comfy() as (base, _state), shell_proxy() as (_proxy, hits):
assert hits["hits"] == 0
ok, info = comfy_common.probe_reachable(base, force=True)
assert ok is True and info == ""
assert comfy_common.require_reachable(base) == base
assert hits["hits"] == 0, "直连之后不该再有任何 ComfyUI 请求被代理截走"
def test_without_the_fix_the_same_endpoint_looks_like_a_gateway_502(shell_proxy, monkeypatch):
"""反证:一旦打开系统代理,同一个地址就会被代理伪造成 502(这就是原来的误判来源)。"""
with fake_comfy() as (base, _state), shell_proxy() as (_proxy, hits):
monkeypatch.setenv("NEX_COMFY_USE_PROXY", "1")
invalidate_reachability()
ok, info = comfy_common.probe_reachable(base, force=True)
assert ok is False and "502" in info, info
assert hits["hits"] >= 1, "代理确实替 ComfyUI 回答了这次请求"
def test_dead_endpoint_is_reported_as_refused_not_as_outage(shell_proxy):
"""端口关着就该说「连接被拒绝」,而不是「HTTP 502 服务抖动」——方向完全不同。"""
with shell_proxy():
dead = _free_base()
with pytest.raises(ProviderError) as caught:
comfy_common.require_reachable(dead, stage="开工前")
message = str(caught.value)
assert caught.value.code == "provider_unreachable"
# 断言前先把地址从文本里摘掉:``_free_base()`` 拿的是随机空闲端口,
# 50227 这种端口号自带「502」三个字,会让这条用例每跑几千次红一次。
haystack = message.replace(dead, "").replace(dead.rsplit(":", 1)[-1], "")
assert "502" not in haystack
assert ("Connection refused" in message or "ConnectError" in message), message
assert "模型管理中心" in message
def test_video_generation_uploads_and_downloads_directly(shell_proxy, tmp_path, monkeypatch):
"""整条出片链路(上传→提交→轮询→下载)都在直连下跑通,代理一次都没被问到。"""
with shell_proxy() as (_proxy, hits), fake_comfy() as (base, state):
result = ComfyVideoProvider().generate(
VideoGenRequest(prompt="雨夜天台,霓虹反光", duration_seconds=6, mode="t2v",
output_path=tmp_path / "shot.mp4"),
{"base_url": base, "poll_interval": 0.05, "timeout": 30},
on_progress=lambda *_a: None)
assert result.provider_job_id == "job-blip-1"
assert state["downloads"] == 1 and tmp_path.joinpath("shot.mp4").stat().st_size > 0
assert hits["hits"] == 0, hits
# ---------------------------------------------------------------- 探测结论
def test_probe_failure_names_the_endpoint_instead_of_dumping_english(shell_proxy):
"""探测失败要给结论:哪个地址、为什么、下一步查什么(而不是 ConnectError 原文)。"""
with shell_proxy():
dead = _free_base()
for adapter in (ComfyVideoProvider(), ComfyImageProvider(), ComfyTTSProvider()):
info = adapter.diagnose({"base_url": dead})
assert info["available"] is False and info.get("reachable") is False, info
message = info["message"]
assert "连不上 ComfyUI" in message and dead in message, message
assert "Connection refused" in message, message
assert "不走系统代理" in message, message
def test_probe_still_reports_the_real_service(shell_proxy):
"""直连改造之后,真服务的探测结论一个字都不该变:连通就是连通。
假 ComfyUI 只挂了 H3 视频需要的节点,所以图像/配音按「可达但缺节点」判定——
这正是 ``probe_reachable`` 与 ``is_available`` 必须分开的原因。
"""
with shell_proxy():
with fake_comfy() as (base, _state):
video = ComfyVideoProvider().diagnose({"base_url": base})
assert video["available"] is True and video["message"].startswith("ComfyUI test"), video
for adapter in (ComfyImageProvider(), ComfyTTSProvider()):
info = adapter.diagnose({"base_url": base})
assert info.get("reachable") is not False, info
assert "连不上" not in info["message"], info
assert "缺少节点" in info["message"], info