323 lines
18 KiB
Python
323 lines
18 KiB
Python
"""回归用例:喂给联合音视频模型(MiniMax H3)的 caption 里不能出现「会被念出来」的整句。
|
||
|
||
现场证据(项目 11《告诉你个小秘密第1期》v2 成片逐镜 ASR):
|
||
提示词开头那几条陈述整句
|
||
|
||
<Picture 1>是这一镜的画面。<Picture 2>是俊杰,高大,和蔼。
|
||
|
||
被模型当成旁白**原样念了出来**(shot-02 前 2.68s、shot-03 前 2.0s 的
|
||
「是这一镜的画面和石腰准和碍」就是它),台词本身反而正常。
|
||
把绑定改成定语写法「<Picture n>中的俊杰」后重新生成(task 541),
|
||
ASR 只剩台词本体、开头完全静音,画面里两个角色的长相/服装仍然与定妆图一致。
|
||
所以这里锁两件事:① 默认写法不留任何「<Picture n>是X」整句;
|
||
② 台词、身份锚定、镜头语言一样都不能少(不能为了静音把信息删光)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
|
||
from app.models import Character, Project, Scene, Shot
|
||
from app.services import prompt as P
|
||
|
||
ROLES = [{"index": 1, "owner_type": "frame", "owner_id": None, "name": "画面"},
|
||
{"index": 2, "owner_type": "human", "owner_id": None, "name": "俊杰"},
|
||
{"index": 3, "owner_type": "human", "owner_id": None, "name": "紫小慧"},
|
||
{"index": 4, "owner_type": "scene", "owner_id": None, "name": "场景"}]
|
||
|
||
|
||
def _case(db):
|
||
project = Project(name="自检-联合caption", aspect_ratio="9:16", resolution="720p", fps=24,
|
||
style_guide="办公室轻松风格", default_provider="comfyui", target_duration=20)
|
||
db.add(project)
|
||
db.flush()
|
||
male = Character(project_id=project.id, name="俊杰", gender="male", appearance="高大,和蔼")
|
||
female = Character(project_id=project.id, name="紫小慧", gender="female",
|
||
appearance="年轻东亚女生,精致淡妆")
|
||
scene = Scene(project_id=project.id, name="办公室", prompt="现代简约的办公室,明亮整洁,有电脑工位")
|
||
db.add_all([male, female, scene])
|
||
db.flush()
|
||
shot = Shot(project_id=project.id, order_index=1, code="S01", title="俊杰解答",
|
||
description="俊杰坐在工位上,从容自信地微笑着转头看向紫小慧",
|
||
duration=8, shot_size="close_up", camera_angle="eye_level",
|
||
camera_movement="static", mode="r2v", status="ready",
|
||
dialogue="不要着急,恢复系统和数据一键就能完成。", speaker_character_id=male.id,
|
||
ambience="办公室环境底噪")
|
||
shot.characters = [male, female]
|
||
shot.scenes = [scene]
|
||
db.add(shot)
|
||
db.flush()
|
||
roles = [dict(ROLES[0]), dict(ROLES[1], owner_id=male.id),
|
||
dict(ROLES[2], owner_id=female.id), dict(ROLES[3], owner_id=scene.id)]
|
||
return project, shot, roles
|
||
|
||
|
||
def test_joint_caption_leaves_no_speakable_binding_sentence(db):
|
||
"""默认(inline):整段没有任何「<Picture n>是X」陈述句,也不以标签开头。"""
|
||
project, shot, roles = _case(db)
|
||
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
|
||
|
||
assert not re.search(r"<Picture \d+>是", text), f"仍有会被念出来的绑定整句:{text}"
|
||
# 每个标签后面都必须是「中的…」定语:定语不是句子,模型不会朗读它
|
||
assert re.findall(r"<Picture \d+>(?!中的)", text) == [], f"有标签没走定语写法:{text}"
|
||
# 身份锚定一句没少,只是换成不会被朗读的定语
|
||
assert "<Picture 2>中的俊杰" in text and "<Picture 3>中的紫小慧" in text
|
||
assert "<Picture 1>中的画面" in text and "<Picture 4>中的" in text
|
||
# 台词是官方说话段落,不再是「{名}台词:「…」」这种能被朗读的说明句
|
||
assert "<d>不要着急,恢复系统和数据一键就能完成。</d>" in text, text
|
||
assert text.endswith("</d>"), f"台词必须是最后一段:{text}"
|
||
assert "台词" not in text and ":" not in text, text
|
||
# 特征短语与动作之间必须留分隔,否则「年轻东亚女生转头看向」会被当成新词
|
||
assert ",," not in text, text
|
||
assert "转头看向<Picture 3>中的紫小慧,年轻东亚女生" in text, text
|
||
# 「和蔼」这类看不见的气质/妆造词不进联合提示词:定妆图已经锁死了人,
|
||
# 气质文字模型既演不出来又最容易被当台词念出来(「轻熟纯欲气质」实测被念)。
|
||
assert "和蔼" not in text and "精致淡妆" not in text, text
|
||
assert "<Picture 2>中的俊杰,高大,坐在工位上" in text, text
|
||
# 镜头语言照旧下发;时长/画幅不再写成文字——它们是 length/width/height 结构化入参,
|
||
# 写成句子就会被念出来(v2 成片开头那句「时长约 8 秒画幅 9 比 16」)
|
||
assert "近景" in text and "固定机位" in text, text
|
||
assert "时长" not in text and "画幅" not in text and "秒" not in text, text
|
||
# 没有参考音色时不能凭空写 <Audio n>:标签指向不存在的素材会串音
|
||
assert "<Audio" not in text, text
|
||
|
||
|
||
def test_joint_caption_only_one_speech_clause(db):
|
||
"""除了台词本体,caption 里不得再出现第二处「说话」相关内容。"""
|
||
project, shot, roles = _case(db)
|
||
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
|
||
speech_words = ("台词", "说", "旁白", "配音", "口型", "语音", "字幕", "背景音", "音色", "BGM")
|
||
hits = [w for w in speech_words if w in text]
|
||
assert hits == [], f"caption 里混进了声音话题 {hits}:{text}"
|
||
|
||
|
||
def test_joint_caption_binds_each_line_to_its_own_voice(db):
|
||
"""``<Audio n>`` 是 H3 里**唯一**合法的「谁用这条嗓子说话」写法。
|
||
|
||
ComfyUI 的 ``MiniMaxH3ReferenceToVideo`` 没有独立 dialogue 字段,文本编码器把
|
||
``<Audio j>:`` 标签和 prompt 裸拼(源码 ``comfy_text_encoders_minimax.py``),
|
||
docstring 明确要求提示词用 ``<Audio j>`` 引用 ``ref_audio_j``。序号按节点输入
|
||
顺序分配,所以多角色必须逐句绑定,不能整体套第一条。
|
||
"""
|
||
project, shot, roles = _case(db)
|
||
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
|
||
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
|
||
db.flush()
|
||
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
|
||
duration_seconds=8, voice_names=["俊杰", "紫小慧"])
|
||
assert "<Audio 2><d>电脑怎么了?</d>" in text, text # 紫小慧 = 第 2 条参考音色
|
||
assert "<Audio 1><d>一键就能恢复。</d>" in text, text # 俊杰 = 第 1 条
|
||
assert text.count("<Audio ") == 2 and text.count("<d>") == 2, text
|
||
# 两段台词之间不要再补句号:那会被当成句子结束、各配一段气口
|
||
assert "</d><Audio 2>" not in text and "</d>。<Audio" not in text, text
|
||
|
||
# 查不到说话人(没配默认音色)时宁可不加标签,也不能绑错嗓子
|
||
blind = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
|
||
duration_seconds=8, voice_names=["", ""])
|
||
assert "<Audio" not in blind and "<d>电脑怎么了?</d>" in blind, blind
|
||
|
||
# 多人对白只收到一条音色时**不能**给第二个人套 <Audio 1>:
|
||
# 标签是硬指令,模型会照着让紫小慧用俊杰的嗓子开口(比不标更糟)。
|
||
thin = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
|
||
duration_seconds=8, voice_names=["俊杰"])
|
||
assert "<Audio 1><d>一键就能恢复。</d>" in thin, thin # 俊杰那句仍然绑自己的嗓子
|
||
assert "<d>电脑怎么了?</d>" in thin and "<Audio 1><d>电脑怎么了?</d>" not in thin, thin
|
||
assert thin.count("<Audio ") == 1, thin
|
||
|
||
# 只有一个说话人的镜头才允许「这条嗓子就是他本人」的兜底(声线没回指角色时名字是空的)
|
||
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
|
||
db.flush()
|
||
solo = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
|
||
duration_seconds=8, voice_names=[""])
|
||
assert "<Audio 1><d>一键就能恢复。</d>" in solo, solo
|
||
|
||
|
||
def _char(project, name: str):
|
||
"""按名字取人物(``project.characters`` 的顺序不保证,别按下标猜)。"""
|
||
return next(c for c in project.characters if str(c.name or "").strip() == name)
|
||
|
||
|
||
def _voiced(db, project):
|
||
"""给两个角色各配一条**可用的**默认声线(``<Audio n>`` 的唯一依据)。
|
||
|
||
只有声线记录还不够:真正下发给 H3 的是声线的克隆样本文件,没有文件时
|
||
``reference_voice_audios`` 会返回空,``<Audio n>`` 与预检告警都按「没绑」处理。
|
||
"""
|
||
import io
|
||
import wave
|
||
|
||
from app.constants import AssetKind
|
||
from app.models import VoiceAsset
|
||
from app.services import storage
|
||
|
||
by_name = {str(c.name or "").strip(): c for c in project.characters}
|
||
male, female = by_name["俊杰"], by_name["紫小慧"]
|
||
frames = io.BytesIO()
|
||
with wave.open(frames, "wb") as handle:
|
||
handle.setnchannels(1)
|
||
handle.setsampwidth(2)
|
||
handle.setframerate(16000)
|
||
handle.writeframes(b"\x20\x00" * 8000)
|
||
payload = frames.getvalue()
|
||
|
||
voices = []
|
||
for char, name, gender in ((male, "俊杰-低音", "male"), (female, "紫小慧-清亮", "female")):
|
||
voice = VoiceAsset(project_id=project.id, name=name, provider="comfyui",
|
||
provider_voice_id=f"v-{char.id}", gender=gender)
|
||
db.add(voice)
|
||
db.flush()
|
||
char.default_voice_id = voice.id
|
||
info = storage.save_bytes(payload, project_id=project.id, kind="audio",
|
||
stem=f"voice-{voice.id}", suffix=".wav")
|
||
storage.register_asset(db, project_id=project.id, owner_type="voice", owner_id=voice.id,
|
||
kind=AssetKind.REFERENCE, path=info["path"],
|
||
name=f"{name}克隆样本", info=info)
|
||
voices.append(voice)
|
||
db.flush()
|
||
return voices
|
||
|
||
|
||
def test_dialogue_voices_follow_the_order_of_lines(db):
|
||
"""参考音色的**顺序**就是 ``<Audio n>`` 的序号,必须由「谁先开口」决定。
|
||
|
||
旧实现只按 ``shot.speaker_character_id`` 收 1 条,再靠「台词文字里出现角色名」补人:
|
||
双人对白镜头(两条 lines、两个 character_id)通常只下发一条参考音频,
|
||
第二个人根本没有自己的 ``<Audio n>``,整段话由第一个人的嗓子念完。
|
||
"""
|
||
project, shot, _ = _case(db)
|
||
male_v, female_v = _voiced(db, project)
|
||
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
|
||
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
|
||
db.flush()
|
||
|
||
got = P.dialogue_voices(db, project, shot)
|
||
assert [v.id for v in got] == [female_v.id, male_v.id], [v.name for v in got]
|
||
|
||
# 反过来开口:顺序跟着换,标签才不会把台词安到别人的嗓子上
|
||
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id},
|
||
{"text": "电脑怎么了?", "character_id": shot.characters[1].id}])
|
||
db.flush()
|
||
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id, female_v.id]
|
||
|
||
# 同一句话被两个人分着说也只会各收一条(去重按声线 id)
|
||
shot.set_lines([{"text": "别慌。", "character_id": shot.characters[0].id},
|
||
{"text": "我看看。", "character_id": shot.characters[0].id},
|
||
{"text": "好。", "character_id": shot.characters[1].id}])
|
||
db.flush()
|
||
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id, female_v.id]
|
||
|
||
|
||
def test_dialogue_voices_falls_back_only_for_unbound_speakers(db):
|
||
"""逐句全都绑好时不再塞镜头级声线;有人没绑才补,免得往参考池里丢无关音色。"""
|
||
from app.models import VoiceAsset
|
||
from app.services import audio
|
||
|
||
project, shot, _ = _case(db)
|
||
male_v, _female_v = _voiced(db, project)
|
||
generic = VoiceAsset(project_id=project.id, name="项目默认声线", provider="comfyui",
|
||
provider_voice_id="v-generic", gender="male", is_default=True)
|
||
db.add(generic)
|
||
db.flush()
|
||
shot.voice_asset_id = generic.id # 老数据:镜头上还固化着一条通用音色
|
||
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
|
||
db.flush()
|
||
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id]
|
||
|
||
# 紫小慧没有默认声线 → 补上镜头级解析结果,别让台词完全失去音色参考
|
||
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id},
|
||
{"text": "电脑怎么了?", "character_id": shot.characters[1].id}])
|
||
project.characters[1].default_voice_id = None
|
||
db.flush()
|
||
got = [v.id for v in P.dialogue_voices(db, project, shot)]
|
||
assert got[0] == male_v.id and audio.resolve_voice_asset(db, project, shot).id in got
|
||
|
||
|
||
def test_joint_unvoiced_speakers_names_every_gap(db):
|
||
"""台词多但参考音色少时,必须点名「哪几个人的嗓子没绑上」——这是静默换声的来源。"""
|
||
project, shot, _ = _case(db)
|
||
male_v, female_v = _voiced(db, project)
|
||
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
|
||
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
|
||
db.flush()
|
||
|
||
assert P.joint_unvoiced_speakers(db, project, shot, voices=[female_v, male_v]) == []
|
||
# 只下发了第一条:紫小慧有自己 <Audio 1>,俊杰那句裸奔
|
||
assert P.joint_unvoiced_speakers(db, project, shot, voices=[female_v]) == ["俊杰"]
|
||
# 一条都没有 / 声线没回指到角色(labels 全是空串)→ 两人都算未绑
|
||
assert P.joint_unvoiced_speakers(db, project, shot, voices=[]) == ["紫小慧", "俊杰"]
|
||
assert P.joint_unvoiced_speakers(db, project, shot, voices=[male_v]) == ["紫小慧"]
|
||
# 没台词的镜头不该报警
|
||
shot.set_lines([])
|
||
shot.dialogue = ""
|
||
db.flush()
|
||
assert P.joint_unvoiced_speakers(db, project, shot, voices=[]) == []
|
||
|
||
|
||
def test_preflight_warns_about_unbound_line_speaker(db, monkeypatch):
|
||
"""素材检查要为「逐句说话人没声线」出 warning:旧检查只看镜头级 speaker,多说话人漏网。"""
|
||
from app.providers.base import Capabilities
|
||
from app.services import preflight, timeline
|
||
|
||
project, shot, _ = _case(db)
|
||
_male_v, female_v = _voiced(db, project)
|
||
female = _char(project, "紫小慧")
|
||
shot.set_lines([{"text": "电脑怎么了?", "character_id": female.id},
|
||
{"text": "一键就能恢复。", "character_id": _char(project, "俊杰").id}])
|
||
female.default_voice_id = None # 只有紫小慧没绑声线,俊杰那条是齐的
|
||
db.flush()
|
||
monkeypatch.setattr(timeline, "native_voice_wanted", lambda *a, **k: True)
|
||
monkeypatch.setattr(preflight.model_registry, "capabilities_of",
|
||
lambda spec: Capabilities(modes=("r2v", "i2v", "t2v"),
|
||
resolutions=("720p",), aspect_ratios=("9:16",),
|
||
min_duration=2.0, max_duration=15.0,
|
||
max_reference_audios=3, joint_av=True))
|
||
issues = {i.code: i for i in preflight.check_shot(db, project, shot).issues}
|
||
assert "joint_speaker_unbound" in issues, set(issues)
|
||
assert "紫小慧" in issues["joint_speaker_unbound"].message, issues["joint_speaker_unbound"]
|
||
assert "俊杰" not in issues["joint_speaker_unbound"].message, issues["joint_speaker_unbound"]
|
||
assert issues["joint_speaker_unbound"].severity == "warning", "漏绑只是会换嗓子,不该挡住派发"
|
||
|
||
# 补上默认声线后告警消失
|
||
female.default_voice_id = female_v.id
|
||
db.flush()
|
||
codes = {i.code for i in preflight.check_shot(db, project, shot).issues}
|
||
assert "joint_speaker_unbound" not in codes, codes
|
||
|
||
|
||
def test_joint_duration_uses_measured_h3_speed(db):
|
||
"""台词时长必须按 H3 实测语速算,且**允许短于**分镜时长。
|
||
|
||
TTS 口径(``audio.CPS_CJK=4.3`` + 标点停顿 + 不低于 shot.duration)给 35 字台词
|
||
留了 9.42s,而 H3 实测 ≈5.0 字/秒 6.8s 就念完——多出来的 2.4s 空档被模型拿来
|
||
念提示词(shot 36 前 2.5s 乱语的直接成因)。对齐还要满足 H3 的帧数格点
|
||
``n % 17 == 5``@24fps,向下取格点。
|
||
"""
|
||
from types import SimpleNamespace
|
||
|
||
project, shot, _ = _case(db)
|
||
caps = SimpleNamespace(min_duration=5.0, max_duration=15.0)
|
||
voices = [SimpleNamespace(tuning={"speed": 1.0})]
|
||
# 「不要着急,恢复系统和数据一键就能完成。」= 19 字 → 19/5 + 0.25 ≈ 4.05s
|
||
short = P.joint_speech_duration_seconds(db, project, shot, caps,
|
||
voices=voices, base=8.0)
|
||
assert short == 5.0, short # 夹到 min_duration
|
||
shot.dialogue = "不要着急,我们公司使用的是紫光汇智的云桌面方案,恢复系统和数据一键就能完成。"
|
||
db.flush()
|
||
got = P.joint_speech_duration_seconds(db, project, shot, caps,
|
||
voices=voices, base=9.42)
|
||
assert 6.5 <= got <= 7.8, got # 35 字 ≈ 7.25s,明显短于 base
|
||
assert got < 9.42, "长于台词的空档会被模型拿来念提示词"
|
||
# 超上限时夹到 max,不能让台词被截断
|
||
shot.dialogue = "很长" * 60
|
||
db.flush()
|
||
assert P.joint_speech_duration_seconds(db, project, shot, caps,
|
||
voices=voices, base=8.0) == 15.0
|
||
|
||
|
||
def test_joint_ref_style_switch_keeps_legacy_form(db, monkeypatch):
|
||
"""留一条回退路:``JOINT_REF_STYLE="prefix"`` 要能还原旧的写法做 A/B。"""
|
||
project, shot, roles = _case(db)
|
||
monkeypatch.setattr(P, "JOINT_REF_STYLE", "prefix")
|
||
legacy = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
|
||
assert legacy.startswith("<Picture 1>是这一镜的画面")
|
||
assert "<Picture 2>是俊杰" in legacy
|