Merge branch 'dev_na' of http://111.10.202.195:3000/chenh/imeeting into dev_ymcg

同步 dev_na 分支的最新更新,确保 dev_ymcg 包含远程修复的功能模块和代码变更,避免后续冲突。
dev_na
puz 2026-06-26 14:14:38 +08:00
commit 255e268138
59 changed files with 3148 additions and 1077 deletions

View File

@ -177,6 +177,12 @@
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId> <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
<version>2.3.0</version> <version>2.3.0</version>
</dependency> </dependency>
<!-- Source: https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-speech-sdk-java -->
<dependency>
<groupId>com.tencentcloudapi</groupId>
<artifactId>tencentcloud-speech-sdk-java</artifactId>
<version>1.0.67</version>
</dependency>
</dependencies> </dependencies>
<build> <build>

View File

@ -71,6 +71,10 @@ public final class RedisKeys {
return "biz:meeting:realtime:state:" + meetingId; return "biz:meeting:realtime:state:" + meetingId;
} }
public static String realtimeMeetingTranscriptCacheKey(Long meetingId) {
return "biz:meeting:realtime:transcript-cache:" + meetingId;
}
public static String realtimeMeetingResumeTimeoutKey(Long meetingId) { public static String realtimeMeetingResumeTimeoutKey(Long meetingId) {
return realtimeMeetingResumeTimeoutPrefix() + meetingId; return realtimeMeetingResumeTimeoutPrefix() + meetingId;
} }

View File

@ -10,6 +10,10 @@ public final class SysParamKeys {
public static final String CAPTCHA_ENABLED = "security.captcha.enabled"; public static final String CAPTCHA_ENABLED = "security.captcha.enabled";
/** AI 会议总结使用的系统提示词。 */ /** AI 会议总结使用的系统提示词。 */
public static final String MEETING_SUMMARY_SYSTEM_PROMPT = "meeting.summary.system_prompt"; public static final String MEETING_SUMMARY_SYSTEM_PROMPT = "meeting.summary.system_prompt";
/** 是否启用 AI 目录。 */
public static final String MEETING_AI_CATALOG_ENABLED = "meeting.ai_catalog.enabled";
/** 会议总结派发模式PARALLEL / SERIAL。 */
public static final String MEETING_SUMMARY_DISPATCH_MODE = "meeting.summary.dispatch_mode";
/** 离线会议音频上传大小上限,单位 MB。 */ /** 离线会议音频上传大小上限,单位 MB。 */
public static final String MEETING_OFFLINE_AUDIO_MAX_SIZE_MB = "meeting.offline_audio.max_size_mb"; public static final String MEETING_OFFLINE_AUDIO_MAX_SIZE_MB = "meeting.offline_audio.max_size_mb";
/** 是否允许创建离线会议。 */ /** 是否允许创建离线会议。 */

View File

@ -36,4 +36,16 @@ public class MeetingAsyncExecutorConfig {
executor.initialize(); executor.initialize();
return executor; return executor;
} }
@Bean("chunkMergeExecutor")
public Executor chunkMergeExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(4);
executor.setQueueCapacity(10);
executor.setThreadNamePrefix("imeeting-chunk-merge-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.initialize();
return executor;
}
} }

View File

@ -49,7 +49,7 @@ public class AndroidMeetingChunkUploadController {
"meetingId", meetingId, "meetingId", meetingId,
"chunkIndex", chunkIndex, "chunkIndex", chunkIndex,
"chunkFile", chunkFile); "chunkFile", chunkFile);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request,true);
androidChunkUploadService.saveChunk(meetingId, chunkIndex, chunkFile, authContext); androidChunkUploadService.saveChunk(meetingId, chunkIndex, chunkFile, authContext);
return ApiResponse.ok(true); return ApiResponse.ok(true);
} }
@ -71,6 +71,7 @@ public class AndroidMeetingChunkUploadController {
"meetingId", meetingId, "meetingId", meetingId,
"totalChunks", totalChunks); "totalChunks", totalChunks);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
return ApiResponse.ok(androidChunkUploadService.completeUpload(meetingId, totalChunks, authContext)); androidChunkUploadService.completeUploadAsync(meetingId, totalChunks, authContext);
return ApiResponse.ok(new LegacyUploadAudioResponse(meetingId, null, "后台合并上传中"));
} }
} }

View File

@ -56,6 +56,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@ -79,6 +80,7 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Tag(name = "Android会议接口") @Tag(name = "Android会议接口")
@ -97,7 +99,7 @@ public class AndroidMeetingController {
private String h5BaseUrl; private String h5BaseUrl;
private final AndroidAuthService androidAuthService; private final AndroidAuthService androidAuthService;
private final AndroidMeetingPushService androidMeetingPushService; private final AndroidMeetingPushService androidMeetingPushService;
private final AndroidChunkUploadService androidChunkUploadService; private final AndroidChunkUploadService androidChunkUploadService;
private final LegacyMeetingAdapterService legacyMeetingAdapterService; private final LegacyMeetingAdapterService legacyMeetingAdapterService;
private final MeetingQueryService meetingQueryService; private final MeetingQueryService meetingQueryService;
@ -148,7 +150,7 @@ public class AndroidMeetingController {
this.paramService = paramService; this.paramService = paramService;
this.dictItemService = dictItemService; this.dictItemService = dictItemService;
this.meetingUnifiedStatusService = meetingUnifiedStatusService; this.meetingUnifiedStatusService = meetingUnifiedStatusService;
this.androidMeetingPushService = androidMeetingPushService; this.androidMeetingPushService = androidMeetingPushService;
} }
@Operation(summary = "创建Android离线会议") @Operation(summary = "创建Android离线会议")
@ -238,22 +240,24 @@ public class AndroidMeetingController {
AndroidRequestLogHelper.logRequest(log, "Android会议", "结束离线会议录音阶段", AndroidRequestLogHelper.logRequest(log, "Android会议", "结束离线会议录音阶段",
"meetingId", meetingId, "meetingId", meetingId,
"request", command); "request", command);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request, true);
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser); requireOperableOfflineMeeting(meetingId, authContext, loginUser);
LegacyUploadAudioResponse uploadResult = null; MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId);
LegacyUploadAudioResponse uploadResult = new LegacyUploadAudioResponse();
if (isUploadFinishedStage(command)) { if (isUploadFinishedStage(command)) {
uploadResult = androidChunkUploadService.completeUpload( androidChunkUploadService.completeUploadAsync(
meeting.getId(), meeting.getId(),
command == null ? null : command.getTotalChunks(), command.getTotalChunks(),
authContext authContext
); );
if (uploadResult == null || uploadResult.getAudioUrl() == null || uploadResult.getAudioUrl().isBlank()) { // if (uploadResult == null) {
throw new RuntimeException("分片上传完成后未生成 audio_url"); // throw new RuntimeException("分片上传完成后未生成结果");
} // }
uploadResult.setMeetingId(meetingId);
} }
meetingCommandService.finishOfflineMeeting(meeting.getId(), command == null ? null : command.getFinishStage()); meetingCommandService.finishOfflineMeeting(meeting.getId(), command == null ? null : command.getFinishStage());
return ApiResponse.ok(uploadResult != null ? uploadResult : true); return ApiResponse.ok(uploadResult);
} }
@Operation(summary = "分页查询Android会议") @Operation(summary = "分页查询Android会议")
@ -292,7 +296,7 @@ public class AndroidMeetingController {
} }
@Operation(summary = "查询Android会议统一状态") @Operation(summary = "查询Android会议统一状态")
@ApiResponses({ @ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse( @io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200", responseCode = "200",
@ -310,13 +314,14 @@ public class AndroidMeetingController {
"request", command); "request", command);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser); requireOperableOfflineMeeting(meetingId, authContext, loginUser);
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId,false);
UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId); UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId);
boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript()); boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript());
boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary()); boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary());
List<MeetingTranscriptVO> transcripts = includeTranscript ? meetingQueryService.getTranscripts(meetingId) : null; List<MeetingTranscriptVO> transcripts = includeTranscript ? meetingQueryService.getTranscripts(meetingId) : null;
String summaryContent = includeSummary ? meetingQueryService.getDetailIgnoreTenant(meetingId).getSummaryContent() : null; String summaryContent = includeSummary ? meeting.getSummaryContent() : null;
return ApiResponse.ok(AndroidUnifiedMeetingStatusResponse.builder() AndroidUnifiedMeetingStatusResponse build = AndroidUnifiedMeetingStatusResponse.builder()
.meetingId(meetingId) .meetingId(meetingId)
.status(status) .status(status)
.meeting(meeting) .meeting(meeting)
@ -324,7 +329,9 @@ public class AndroidMeetingController {
.transcripts(transcripts) .transcripts(transcripts)
.includesSummary(includeSummary) .includesSummary(includeSummary)
.summaryContent(summaryContent) .summaryContent(summaryContent)
.build()); .build();
log.info("[{}]{}.返回数据:[{}]", "Android会议", "查询会议统一状态", build);
return ApiResponse.ok(build);
} }
@Operation(summary = "重试 Android 会议 ASR 识别") @Operation(summary = "重试 Android 会议 ASR 识别")
@ -343,7 +350,7 @@ public class AndroidMeetingController {
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
requireOperableOfflineMeeting(meetingId, authContext, loginUser); requireOperableOfflineMeeting(meetingId, authContext, loginUser);
meetingCommandService.retryTranscription(meetingId); meetingCommandService.retryTranscription(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode()); androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode());
return ApiResponse.ok(true); return ApiResponse.ok(true);
} }
@ -364,17 +371,18 @@ public class AndroidMeetingController {
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
requireOperableOfflineMeeting(meetingId, authContext, loginUser); requireOperableOfflineMeeting(meetingId, authContext, loginUser);
meetingCommandService.retrySummary(meetingId); meetingCommandService.retrySummary(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode()); androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode());
return ApiResponse.ok(true); return ApiResponse.ok(true);
} }
@Operation(summary = "更新Android会议访问密码")
@ApiResponses({ @Operation(summary = "更新Android会议访问密码")
@io.swagger.v3.oas.annotations.responses.ApiResponse( @ApiResponses({
responseCode = "200", @io.swagger.v3.oas.annotations.responses.ApiResponse(
description = "返回更新后的会议访问密码,传空时表示清空访问密码", responseCode = "200",
content = @Content(schema = @Schema(implementation = String.class)) description = "返回更新后的会议访问密码,传空时表示清空访问密码",
) content = @Content(schema = @Schema(implementation = String.class))
}) )
})
@PutMapping("/{meetingId}/access-password") @PutMapping("/{meetingId}/access-password")
@Log(value = "修改Android会议访问密码", type = "Android会议管理") @Log(value = "修改Android会议访问密码", type = "Android会议管理")
public ApiResponse<String> updateAccessPassword(HttpServletRequest request, public ApiResponse<String> updateAccessPassword(HttpServletRequest request,
@ -390,9 +398,9 @@ public class AndroidMeetingController {
return ApiResponse.error("仅会议创建人可设置访问密码"); return ApiResponse.error("仅会议创建人可设置访问密码");
} }
String password = normalizePassword(command == null ? null : command.getPassword()); String password = normalizePassword(command == null ? null : command.getPassword());
meetingService.update(new LambdaUpdateWrapper<Meeting>() meetingService.update(new LambdaUpdateWrapper<Meeting>()
.eq(Meeting::getId,meeting.getId()) .eq(Meeting::getId, meeting.getId())
.set(Meeting::getAccessPassword, password)); .set(Meeting::getAccessPassword, password));
return ApiResponse.ok(password); return ApiResponse.ok(password);
} }
@ -415,54 +423,55 @@ public class AndroidMeetingController {
meetingCommandService.deleteMeeting(meetingId); meetingCommandService.deleteMeeting(meetingId);
return ApiResponse.ok(true); return ApiResponse.ok(true);
} }
@GetMapping("/config")
@Log(value = "获取会议配置", type = "Android会议管理")
@Operation(summary = "获取会议配置")
@Anonymous
public ApiResponse<AndroidMeetingConfigVo> config(HttpServletRequest request) {
AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口");
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext);
Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId();
Long userId = loginUser != null ? loginUser.getUserId() : null;
boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin());
boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin());
AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo();
PageResult<List<PromptTemplateVO>> promptTemplateList = promptTemplateService.pageTemplates(
1,
1000,
null,
null,
tenantId,
userId,
isPlatformAdmin,
isTenantAdmin
);
List<PromptTemplateVO> enabledTemplates = promptTemplateList.getRecords() == null
? List.of()
: promptTemplateList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setTemplateList(enabledTemplates);
PageResult<List<AiModelVO>> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId);
List<AiModelVO> enabledModels = modelList.getRecords() == null
? List.of()
: modelList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setModelsList(enabledModels);
resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail"));
resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION,"30")));
resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10")));
resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION,String.valueOf(60*4))));
BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99"));
bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP);
resultVo.setPacketLossRate(bigDecimal );
resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false")));
resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60")));
return ApiResponse.ok(resultVo); @GetMapping("/config")
} @Log(value = "获取会议配置", type = "Android会议管理")
@Operation(summary = "获取会议配置")
@Anonymous
public ApiResponse<AndroidMeetingConfigVo> config(HttpServletRequest request) {
AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口");
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext);
Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId();
Long userId = loginUser != null ? loginUser.getUserId() : null;
boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin());
boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin());
AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo();
PageResult<List<PromptTemplateVO>> promptTemplateList = promptTemplateService.pageTemplates(
1,
1000,
null,
null,
tenantId,
userId,
isPlatformAdmin,
isTenantAdmin
);
List<PromptTemplateVO> enabledTemplates = promptTemplateList.getRecords() == null
? List.of()
: promptTemplateList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setTemplateList(enabledTemplates);
PageResult<List<AiModelVO>> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId);
List<AiModelVO> enabledModels = modelList.getRecords() == null
? List.of()
: modelList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setModelsList(enabledModels);
resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail"));
resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION, "30")));
resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10")));
resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, String.valueOf(60 * 4))));
BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99"));
bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP);
resultVo.setPacketLossRate(bigDecimal);
resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false")));
resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60")));
return ApiResponse.ok(resultVo);
}
private void resolvePublicDeviceTenantId(HttpServletRequest request, private void resolvePublicDeviceTenantId(HttpServletRequest request,
AndroidOfflineMeetingCreateCommand command, AndroidOfflineMeetingCreateCommand command,
@ -545,29 +554,14 @@ public class AndroidMeetingController {
return ChronoUnit.DAYS.between(LocalDate.now(), meetingTime.toLocalDate()); return ChronoUnit.DAYS.between(LocalDate.now(), meetingTime.toLocalDate());
} }
private MeetingVO requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) { private Meeting requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) {
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId); Meeting meeting = meetingAccessService.requireMeetingIgnoreTenant(meetingId);
if (meeting == null) {
throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(), "会议不存在");
}
if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) { if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) {
throw new RuntimeException("当前会议不是离线会议"); throw new RuntimeException("当前会议不是离线会议");
} }
if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) { if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) {
throw new RuntimeException("设备ID不能为空"); throw new RuntimeException("设备ID不能为空");
} }
// if (meeting.getSourceDeviceCode() == null || !meeting.getSourceDeviceCode().equals(authContext.getDeviceId())) {
// throw new RuntimeException("当前会议不属于该设备");
// }
// if (authContext.isAnonymous()) {
// if (!MeetingConstants.DEVICE_MODE_PUBLIC.equals(meeting.getSourceDeviceMode())) {
// throw new RuntimeException("当前会议不是公有设备会议");
// }
// return meeting;
// }
// if (loginUser == null || !Objects.equals(meeting.getCreatorId(), loginUser.getUserId())) {
// throw new RuntimeException("仅会议创建人可操作当前会议");
// }
return meeting; return meeting;
} }

View File

@ -3,6 +3,7 @@ package com.imeeting.controller.biz;
import com.imeeting.dto.biz.AiLocalProfileVO; import com.imeeting.dto.biz.AiLocalProfileVO;
import com.imeeting.dto.biz.AiModelDTO; import com.imeeting.dto.biz.AiModelDTO;
import com.imeeting.dto.biz.AiModelVO; import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.AiModelService; import com.imeeting.service.biz.AiModelService;
import com.unisbase.common.ApiResponse; import com.unisbase.common.ApiResponse;
import com.unisbase.common.annotation.Log; import com.unisbase.common.annotation.Log;
@ -121,7 +122,7 @@ public class AiModelController {
@PostMapping("/llm-connectivity-test") @PostMapping("/llm-connectivity-test")
@PreAuthorize("isAuthenticated()") @PreAuthorize("isAuthenticated()")
public ApiResponse<Boolean> testLlmConnectivity(@RequestBody AiModelDTO dto) { public ApiResponse<Boolean> testLlmConnectivity(@RequestBody AiModelDTO dto) {
if ("custom".equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) { if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) {
return ApiResponse.error("基础地址不能为空"); return ApiResponse.error("基础地址不能为空");
} }
if (dto.getModelCode() == null || dto.getModelCode().isBlank()) { if (dto.getModelCode() == null || dto.getModelCode().isBlank()) {

View File

@ -212,6 +212,7 @@ public class MeetingController {
MeetingCreateConfigVO vo = new MeetingCreateConfigVO(); MeetingCreateConfigVO vo = new MeetingCreateConfigVO();
vo.setOfflineEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_OFFLINE_ENABLED, true)); vo.setOfflineEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_OFFLINE_ENABLED, true));
vo.setRealtimeEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_REALTIME_ENABLED, true)); vo.setRealtimeEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_REALTIME_ENABLED, true));
vo.setAiCatalogEnabled(resolveBooleanParam(SysParamKeys.MEETING_AI_CATALOG_ENABLED, false));
vo.setOfflineAudioMaxSizeMb(resolveLongParam(SysParamKeys.MEETING_OFFLINE_AUDIO_MAX_SIZE_MB, 1024L)); vo.setOfflineAudioMaxSizeMb(resolveLongParam(SysParamKeys.MEETING_OFFLINE_AUDIO_MAX_SIZE_MB, 1024L));
return ApiResponse.ok(vo); return ApiResponse.ok(vo);
} }
@ -398,17 +399,6 @@ public class MeetingController {
return ApiResponse.ok(result); return ApiResponse.ok(result);
} }
@Operation(summary = "追加实时转写片段")
@PostMapping("/{id}/realtime/transcripts")
@PreAuthorize("isAuthenticated()")
public ApiResponse<Boolean> appendRealtimeTranscripts(@PathVariable Long id, @RequestBody List<RealtimeTranscriptItemDTO> items) {
LoginUser loginUser = currentLoginUser();
Meeting meeting = meetingAccessService.requireMeeting(id);
meetingAccessService.assertCanControlRealtimeMeeting(meeting, loginUser, MeetingConstants.SOURCE_WEB);
meetingCommandService.appendRealtimeTranscripts(id, items);
return ApiResponse.ok(true);
}
@Operation(summary = "暂停实时会议") @Operation(summary = "暂停实时会议")
@PostMapping("/{id}/realtime/pause") @PostMapping("/{id}/realtime/pause")
@PreAuthorize("isAuthenticated()") @PreAuthorize("isAuthenticated()")

View File

@ -6,12 +6,14 @@ import com.imeeting.dto.biz.UnifiedMeetingStatusVO;
import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.ToString;
import java.util.List; import java.util.List;
@Data @Data
@Builder @Builder
@Schema(description = "Android 统一会议状态响应") @Schema(description = "Android 统一会议状态响应")
@ToString
public class AndroidUnifiedMeetingStatusResponse { public class AndroidUnifiedMeetingStatusResponse {
@Schema(description = "会议ID") @Schema(description = "会议ID")
private Long meetingId; private Long meetingId;

View File

@ -18,4 +18,12 @@ public class LegacyUploadAudioResponse {
@JsonProperty("audio_url") @JsonProperty("audio_url")
@Schema(description = "上传后的音频访问地址") @Schema(description = "上传后的音频访问地址")
private String audioUrl; private String audioUrl;
@JsonProperty("message")
@Schema(description = "结果提示")
private String message;
public LegacyUploadAudioResponse(Long meetingId, String audioUrl) {
this(meetingId, audioUrl, null);
}
} }

View File

@ -13,6 +13,9 @@ public class MeetingCreateConfigVO {
@Schema(description = "是否启用实时会议") @Schema(description = "是否启用实时会议")
private Boolean realtimeEnabled; private Boolean realtimeEnabled;
@Schema(description = "是否启用 AI 目录")
private Boolean aiCatalogEnabled;
@Schema(description = "离线音频上传大小上限,单位 MB") @Schema(description = "离线音频上传大小上限,单位 MB")
private Long offlineAudioMaxSizeMb; private Long offlineAudioMaxSizeMb;
} }

View File

@ -75,6 +75,9 @@ public class MeetingVO {
@Schema(description = "总结模板ID") @Schema(description = "总结模板ID")
private Long promptId; private Long promptId;
@Schema(description = "是否启用 AI 目录")
private Boolean aiCatalogEnabled;
@Schema(description = "音频保存状态") @Schema(description = "音频保存状态")
private String audioSaveStatus; private String audioSaveStatus;

View File

@ -27,4 +27,6 @@ public class RealtimeMeetingResumeConfig {
private Boolean saveAudio; private Boolean saveAudio;
@Schema(description = "热词列表") @Schema(description = "热词列表")
private List<Map<String, Object>> hotwords; private List<Map<String, Object>> hotwords;
@Schema(description = "腾讯说话人上下文 ID")
private String speakerContextId;
} }

View File

@ -0,0 +1,22 @@
package com.imeeting.dto.biz;
import lombok.Data;
@Data
public class RealtimeMeetingTranscriptCacheItem {
private String sentenceKey;
private String sentenceGroupKey;
private Integer sentenceId;
private Integer sentenceType;
private String speakerId;
private String speakerName;
private String userId;
private Integer startTime;
private Integer endTime;
private String content;
private Integer sortOrder;
private Boolean finalResult;
private Long transcriptId;
private Long firstReceivedAt;
private Long updatedAt;
}

View File

@ -0,0 +1,15 @@
package com.imeeting.dto.biz;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class RealtimeMeetingTranscriptCacheState {
private Long meetingId;
private Integer nextSortOrder;
private Integer nextLegacySequence;
private List<RealtimeMeetingTranscriptCacheItem> items = new ArrayList<>();
private Long updatedAt;
}

View File

@ -2,11 +2,16 @@ package com.imeeting.dto.biz;
import lombok.Data; import lombok.Data;
import java.util.Map;
@Data @Data
public class RealtimeSocketSessionData { public class RealtimeSocketSessionData {
private Long meetingId; private Long meetingId;
private Long userId; private Long userId;
private Long tenantId; private Long tenantId;
private Long asrModelId; private Long asrModelId;
private String provider;
private String targetWsUrl; private String targetWsUrl;
private String modelCode;
private Map<String, Object> mediaConfig;
} }

View File

@ -0,0 +1,17 @@
package com.imeeting.enums;
import lombok.Getter;
@Getter
public enum ModelProviderEnum {
LOCAL("local", "本地"),
TENCENT("tencent", "腾讯云");
private final String code;
private final String description;
ModelProviderEnum(String code, String description) {
this.code = code;
this.description = description;
}
}

View File

@ -254,7 +254,20 @@ public class AndroidPushGrpcService extends PushServiceGrpc.PushServiceImplBase
return switch (platform) { return switch (platform) {
case IOS -> "ios"; case IOS -> "ios";
case ANDROID -> "android"; case ANDROID -> "android";
case PLATFORM_UNKNOWN, UNRECOGNIZED -> "android"; case PLATFORM_UNKNOWN,UNRECOGNIZED -> "android";
case HARMONY_MOBILE ->"harmony_mobile";
// Desktop
case WINDOWS ->"windows";
case MACOS->"macos";
case LINUX -> "linux";
// Linux发行版可选
case KYLIN ->"kylin";
case UOS ->"uos";
// Harmony PC
case HARMONY_PC ->"harmony_pc";
}; };
} }
} }

View File

@ -15,4 +15,12 @@ public interface AndroidChunkUploadService {
LegacyUploadAudioResponse completeUpload(Long meetingId, LegacyUploadAudioResponse completeUpload(Long meetingId,
Integer totalChunks, Integer totalChunks,
AndroidAuthContext authContext) throws IOException; AndroidAuthContext authContext) throws IOException;
/**
* + + 线
* 线Tomcat failOfflineTranscription
*/
void completeUploadAsync(Long meetingId,
Integer totalChunks,
AndroidAuthContext authContext);
} }

View File

@ -5,14 +5,16 @@ import com.imeeting.dto.android.AndroidChunkUploadSessionState;
import com.imeeting.dto.android.legacy.LegacyUploadAudioResponse; import com.imeeting.dto.android.legacy.LegacyUploadAudioResponse;
import com.imeeting.service.android.AndroidChunkUploadService; import com.imeeting.service.android.AndroidChunkUploadService;
import com.imeeting.service.android.legacy.LegacyMeetingAdapterService; import com.imeeting.service.android.legacy.LegacyMeetingAdapterService;
import com.imeeting.service.biz.MeetingCommandService;
import com.imeeting.service.biz.MeetingQueryService;
import com.imeeting.support.TaskSecurityContextRunner;
import com.imeeting.support.redis.AndroidChunkUploadSessionCache; import com.imeeting.support.redis.AndroidChunkUploadSessionCache;
import com.unisbase.security.LoginUser; import lombok.extern.slf4j.Slf4j;
import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@ -30,12 +32,16 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@Service @Service
@RequiredArgsConstructor @Slf4j
public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService { public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService {
private static final Pattern CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$"); private static final Pattern LEGACY_CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$");
private static final Pattern CHUNK_DIR_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)$");
private static final String CHUNK_ROOT_DIR = "chunks";
private final TaskSecurityContextRunner taskSecurityContextRunner;
private final AndroidChunkUploadSessionCache sessionCache; private final AndroidChunkUploadSessionCache sessionCache;
private final LegacyMeetingAdapterService legacyMeetingAdapterService; private final LegacyMeetingAdapterService legacyMeetingAdapterService;
private final MeetingCommandService meetingCommandService;
private final java.util.concurrent.Executor chunkMergeExecutor;
@Value("${unisbase.app.upload-path}") @Value("${unisbase.app.upload-path}")
private String uploadPath; private String uploadPath;
@ -43,6 +49,18 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
@Value("${imeeting.audio.ffmpeg-path:ffmpeg}") @Value("${imeeting.audio.ffmpeg-path:ffmpeg}")
private String ffmpegPath; private String ffmpegPath;
public AndroidChunkUploadServiceImpl(AndroidChunkUploadSessionCache sessionCache,
LegacyMeetingAdapterService legacyMeetingAdapterService,
MeetingCommandService meetingCommandService,
@Qualifier("chunkMergeExecutor") java.util.concurrent.Executor chunkMergeExecutor,
TaskSecurityContextRunner taskSecurityContextRunner) {
this.sessionCache = sessionCache;
this.legacyMeetingAdapterService = legacyMeetingAdapterService;
this.meetingCommandService = meetingCommandService;
this.chunkMergeExecutor = chunkMergeExecutor;
this.taskSecurityContextRunner = taskSecurityContextRunner;
}
@Override @Override
public void saveChunk(Long meetingId, public void saveChunk(Long meetingId,
Integer chunkIndex, Integer chunkIndex,
@ -54,7 +72,7 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (chunkIndex == null || chunkIndex < 0) { if (chunkIndex == null || chunkIndex < 0) {
throw new RuntimeException("分片参数无效"); throw new RuntimeException("分片参数无效");
} }
if (chunkFile == null) { if (chunkFile == null) {
throw new RuntimeException("chunk_file不能为空"); throw new RuntimeException("chunk_file不能为空");
} }
@ -63,19 +81,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
throw new RuntimeException("分片上传会话与当前设备或会议不匹配"); throw new RuntimeException("分片上传会话与当前设备或会议不匹配");
} }
String chunkFileName = buildStoredChunkFileName(chunkIndex, chunkFile.getOriginalFilename()); String originalFileName = resolveOriginalFileName(chunkFile.getOriginalFilename());
String previousFileName = state.getChunkFileNames().get(chunkIndex); Path chunkFilePath = resolveChunkFilePath(meetingId, chunkIndex, originalFileName);
Path meetingDir = sessionDir(meetingId); Path chunkDir = chunkFilePath.getParent();
Files.createDirectories(meetingDir); clearChunkDirectory(chunkDir);
Files.createDirectories(chunkDir);
if (previousFileName != null && !previousFileName.equals(chunkFileName)) { Files.write(chunkFilePath, chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
deleteQuietly(meetingDir.resolve(previousFileName));
String previousFileName = state.getChunkFileNames().put(chunkIndex, originalFileName);
if (previousFileName != null && !previousFileName.equals(originalFileName)) {
state.getUploadedChunkFileNames().remove(previousFileName); state.getUploadedChunkFileNames().remove(previousFileName);
} }
state.getUploadedChunkFileNames().add(originalFileName);
Files.write(meetingDir.resolve(chunkFileName), chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
state.getUploadedChunkFileNames().add(chunkFileName);
state.getChunkFileNames().put(chunkIndex, chunkFileName);
state.getReceivedChunks().add(chunkIndex); state.getReceivedChunks().add(chunkIndex);
saveState(meetingId, state); saveState(meetingId, state);
} }
@ -91,53 +109,70 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
throw new RuntimeException("total_chunks不能为空且必须大于0"); throw new RuntimeException("total_chunks不能为空且必须大于0");
} }
AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext); AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext);
if (!Objects.equals(state.getMeetingId(), meetingId) || !Objects.equals(state.getDeviceId(), authContext.getDeviceId())) { if (!Objects.equals(state.getMeetingId(), meetingId) || !Objects.equals(state.getDeviceId(), authContext.getDeviceId())) {
throw new RuntimeException("分片上传会话与当前设备或会议不匹配"); throw new RuntimeException("分片上传会话与当前设备或会议不匹配");
} }
Path meetingDir = sessionDir(meetingId); Path meetingDir = sessionDir(meetingId);
Files.createDirectories(meetingDir); Files.createDirectories(meetingDir);
state.setTotalChunks(totalChunks); state.setTotalChunks(totalChunks);
List<Path> orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks); List<Path> orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks);
saveState(meetingId, state); saveState(meetingId, state);
Path mergedFile = mergeChunks(state, orderedChunkPaths); Path mergedFile = mergeChunks(state, orderedChunkPaths);
if (mergedFile == null) {
meetingCommandService.failOfflineTranscription(meetingId, "安卓上传文件为空");
cleanup(meetingId);
return new LegacyUploadAudioResponse(meetingId, null, "无可合并音频");
}
MultipartFile mergedMultipart = new LocalMultipartFile( MultipartFile mergedMultipart = new LocalMultipartFile(
buildMergedOriginalFilename(state, mergedFile), resolveMergedOriginalFilename(state, orderedChunkPaths, mergedFile),
state.getContentType(), state.getContentType(),
Files.readAllBytes(mergedFile) mergedFile
); );
LegacyUploadAudioResponse response; LegacyUploadAudioResponse response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice(
if (authContext.isAnonymous()) { meetingId,
response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice( null,
meetingId, null,
null, false,
null, mergedMultipart,
false, authContext
mergedMultipart, );
authContext
);
} else {
LoginUser loginUser = toLoginUser(authContext);
response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcess(
meetingId,
null,
null,
false,
mergedMultipart,
authContext,
loginUser
);
}
if (response != null) { if (response != null) {
cleanup(meetingId); cleanup(meetingId);
} }
return response; return response;
} }
@Override
public void completeUploadAsync(Long meetingId,
Integer totalChunks,
AndroidAuthContext authContext) {
if (meetingId == null) {
throw new RuntimeException("meeting_id不能为空");
}
if (totalChunks == null || totalChunks <= 0) {
throw new RuntimeException("total_chunks不能为空且必须大于0");
}
chunkMergeExecutor.execute( ()->taskSecurityContextRunner.runAsTenantUser( authContext.getTenantId(), authContext.getUserId(), () -> {
try {
completeUpload(meetingId, totalChunks, authContext);
} catch (Exception ex) {
log.error("[分片合并] 会议{}异步合并上传失败: {}", meetingId, ex.getMessage(), ex);
try {
meetingCommandService.failOfflineTranscription(meetingId, "音频合并上传失败: " + ex.getMessage());
} catch (Exception inner) {
log.error("[分片合并] 会议{}标记失败状态异常: {}", meetingId, inner.getMessage(), inner);
}
}
}));
}
private AndroidChunkUploadSessionState getOrCreateState(Long meetingId, private AndroidChunkUploadSessionState getOrCreateState(Long meetingId,
MultipartFile chunkFile, MultipartFile chunkFile,
AndroidAuthContext authContext) { AndroidAuthContext authContext) {
@ -148,155 +183,170 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
AndroidChunkUploadSessionState state = new AndroidChunkUploadSessionState(); AndroidChunkUploadSessionState state = new AndroidChunkUploadSessionState();
state.setMeetingId(meetingId); state.setMeetingId(meetingId);
state.setDeviceId(authContext.getDeviceId()); state.setDeviceId(authContext.getDeviceId());
state.setFileName(normalizeChunkSourceFileName(chunkFile.getOriginalFilename())); state.setFileName(resolveOriginalFileName(chunkFile.getOriginalFilename()));
state.setContentType(chunkFile.getContentType()); state.setContentType(chunkFile.getContentType());
saveState(meetingId, state); saveState(meetingId, state);
return state; return state;
} }
private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) { private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) {
AndroidChunkUploadSessionState state = getState(meetingId); AndroidChunkUploadSessionState state = getState(meetingId);
if (state != null) { if (state != null) {
return state; return state;
} }
AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState(); AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState();
rebuiltState.setMeetingId(meetingId); rebuiltState.setMeetingId(meetingId);
rebuiltState.setDeviceId(authContext.getDeviceId()); rebuiltState.setDeviceId(authContext.getDeviceId());
return rebuiltState; return rebuiltState;
}
private List<Path> rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state,
Path meetingDir,
int totalChunks) throws IOException {
Map<Integer, Path> validatedChunkFiles = scanChunkFiles(meetingDir, true);
Map<Integer, Path> mergeChunkFiles = scanChunkFiles(meetingDir, false);
state.getReceivedChunks().clear();
state.getUploadedChunkFileNames().clear();
state.getChunkFileNames().clear();
for (Map.Entry<Integer, Path> entry : validatedChunkFiles.entrySet()) {
Integer chunkIndex = entry.getKey();
Path stateChunkPath = mergeChunkFiles.getOrDefault(chunkIndex, entry.getValue());
String fileName = stateChunkPath.getFileName().toString();
state.getReceivedChunks().add(chunkIndex);
state.getUploadedChunkFileNames().add(fileName);
state.getChunkFileNames().put(chunkIndex, fileName);
} }
List<Path> orderedChunkPaths = new ArrayList<>(totalChunks); private List<Path> rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state,
for (int i = 0; i < totalChunks; i++) { Path meetingDir,
Path validatedChunkPath = validatedChunkFiles.get(i); int totalChunks) throws IOException {
if (validatedChunkPath == null) { Map<Integer, Path> chunkFiles = scanChunkFiles(meetingDir, true);
throw new RuntimeException("分片未上传完整"); state.getReceivedChunks().clear();
} state.getUploadedChunkFileNames().clear();
Path chunkPath = mergeChunkFiles.get(i); state.getChunkFileNames().clear();
if (chunkPath != null) {
orderedChunkPaths.add(chunkPath);
}
}
return orderedChunkPaths;
}
private Map<Integer, Path> scanChunkFiles(Path meetingDir, boolean includePending) throws IOException { for (Map.Entry<Integer, Path> entry : chunkFiles.entrySet()) {
Map<Integer, Path> chunkFiles = new TreeMap<>(); Integer chunkIndex = entry.getKey();
if (!Files.exists(meetingDir)) { Path chunkPath = entry.getValue();
return chunkFiles; String fileName = chunkPath.getFileName().toString();
} state.getReceivedChunks().add(chunkIndex);
try (var paths = Files.list(meetingDir)) { state.getUploadedChunkFileNames().add(fileName);
paths.filter(Files::isRegularFile) state.getChunkFileNames().put(chunkIndex, fileName);
.filter(path -> isUsableChunkFile(path, includePending)) }
.forEach(path -> {
Integer chunkIndex = parseChunkIndex(path.getFileName().toString(), includePending);
if (chunkIndex == null) {
return;
}
Path existing = chunkFiles.get(chunkIndex);
if (existing == null || isPreferredChunkFile(path, existing)) {
chunkFiles.put(chunkIndex, path);
}
});
}
return chunkFiles;
}
private boolean isUsableChunkFile(Path path, boolean includePending) { List<Path> orderedChunkPaths = new ArrayList<>(totalChunks);
if (path == null || path.getFileName() == null) { for (int i = 0; i < totalChunks; i++) {
return false; Path chunkPath = chunkFiles.get(i);
if (chunkPath == null) {
throw new RuntimeException("分片未上传完整");
}
orderedChunkPaths.add(chunkPath);
}
return orderedChunkPaths;
} }
String fileName = normalizeChunkStorageFileName(path.getFileName().toString(), includePending);
if (fileName == null) {
return false;
}
return CHUNK_FILE_NAME_PATTERN.matcher(fileName).matches();
}
private Integer parseChunkIndex(String fileName, boolean includePending) { private Map<Integer, Path> scanChunkFiles(Path meetingDir, boolean includePending) throws IOException {
String normalizedFileName = normalizeChunkStorageFileName(fileName, includePending); Map<Integer, Path> chunkFiles = new TreeMap<>();
if (normalizedFileName == null) { if (!Files.exists(meetingDir)) {
return null; return chunkFiles;
} }
Matcher matcher = CHUNK_FILE_NAME_PATTERN.matcher(normalizedFileName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
}
private boolean isPreferredChunkFile(Path candidate, Path existing) { Path chunkRoot = meetingDir.resolve(CHUNK_ROOT_DIR);
boolean candidatePending = isPendingChunkFile(candidate); if (Files.exists(chunkRoot)) {
boolean existingPending = isPendingChunkFile(existing); try (var dirs = Files.list(chunkRoot)) {
if (candidatePending != existingPending) { dirs.filter(Files::isDirectory).forEach(dir -> {
return !candidatePending; Integer chunkIndex = parseChunkDirIndex(dir.getFileName().toString());
} if (chunkIndex == null || chunkFiles.containsKey(chunkIndex)) {
try { return;
return Files.getLastModifiedTime(candidate).compareTo(Files.getLastModifiedTime(existing)) >= 0; }
} catch (IOException ex) { Path chunkPath = pickChunkFile(dir, includePending);
return candidate.getFileName().toString().compareTo(existing.getFileName().toString()) >= 0; if (chunkPath != null) {
} chunkFiles.put(chunkIndex, chunkPath);
} }
});
}
if (!chunkFiles.isEmpty()) {
return chunkFiles;
}
}
private String normalizeChunkStorageFileName(String fileName, boolean includePending) { try (var paths = Files.list(meetingDir)) {
if (fileName == null) { paths.filter(Files::isRegularFile)
return null; .forEach(path -> {
Integer chunkIndex = parseLegacyChunkIndex(path.getFileName().toString());
if (chunkIndex != null) {
chunkFiles.put(chunkIndex, path);
}
});
}
return chunkFiles;
} }
if (fileName.endsWith(".pending")) {
return includePending ? fileName.substring(0, fileName.length() - ".pending".length()) : null; private Integer parseChunkDirIndex(String directoryName) {
if (directoryName == null) {
return null;
}
Matcher matcher = CHUNK_DIR_NAME_PATTERN.matcher(directoryName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
} }
return fileName;
}
private boolean isPendingChunkFile(Path path) { private Integer parseLegacyChunkIndex(String fileName) {
return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending"); if (fileName == null) {
} return null;
}
Matcher matcher = LEGACY_CHUNK_FILE_NAME_PATTERN.matcher(fileName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
}
private Path mergeChunks(AndroidChunkUploadSessionState state, List<Path> chunkPaths) throws IOException { private Path pickChunkFile(Path chunkDir, boolean includePending) {
if (chunkDir == null || !Files.isDirectory(chunkDir)) {
return null;
}
try (var files = Files.list(chunkDir)) {
Path preferred = files
.filter(Files::isRegularFile)
.filter(path -> includePending || !isPendingChunkFile(path))
.findFirst()
.orElse(null);
if (preferred != null) {
return preferred;
}
} catch (IOException ex) {
return null;
}
try (var files = Files.list(chunkDir)) {
return files.filter(Files::isRegularFile).findFirst().orElse(null);
} catch (IOException ex) {
return null;
}
}
private Path mergeChunks(AndroidChunkUploadSessionState state, List<Path> chunkPaths) throws IOException {
List<Path> mergeableChunkPaths = filterMergeableChunkPaths(chunkPaths);
Path meetingDir = sessionDir(state.getMeetingId()); Path meetingDir = sessionDir(state.getMeetingId());
String mergedExtension = resolveMergedExtension(state, chunkPaths); String mergedExtension = resolveMergedExtension(state, mergeableChunkPaths);
Path mergedOutput = meetingDir.resolve("merged" + mergedExtension); Path mergedOutput = meetingDir.resolve("merged" + mergedExtension);
Path concatList = meetingDir.resolve("concat-inputs.txt"); Path concatList = meetingDir.resolve("concat-inputs.txt");
Files.deleteIfExists(mergedOutput); Files.deleteIfExists(mergedOutput);
if (chunkPaths.isEmpty() || allChunkFilesEmpty(chunkPaths)) { if (mergeableChunkPaths.isEmpty() || allChunkFilesEmpty(mergeableChunkPaths)) {
Files.write(mergedOutput, new byte[0], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); return null;
return mergedOutput; }
}
if (chunkPaths.size() == 1) { if (mergeableChunkPaths.size() == 1) {
return chunkPaths.get(0); return mergeableChunkPaths.get(0);
} }
writeConcatListFile(concatList, chunkPaths); writeConcatListFile(concatList, mergeableChunkPaths);
executeFfmpegConcat(concatList, mergedOutput); executeFfmpegConcat(concatList, mergedOutput);
return mergedOutput; return mergedOutput;
} }
private boolean allChunkFilesEmpty(List<Path> chunkPaths) throws IOException { private boolean allChunkFilesEmpty(List<Path> chunkPaths) throws IOException {
for (Path chunkPath : chunkPaths) { for (Path chunkPath : chunkPaths) {
if (Files.size(chunkPath) > 0) { if (Files.size(chunkPath) > 0) {
return false; return false;
} }
}
return true;
} }
return true;
private List<Path> filterMergeableChunkPaths(List<Path> chunkPaths) {
if (chunkPaths == null || chunkPaths.isEmpty()) {
return List.of();
}
return chunkPaths.stream()
.filter(path -> path != null && !isPendingChunkFile(path))
.toList();
} }
private AndroidChunkUploadSessionState getState(Long meetingId) { private AndroidChunkUploadSessionState getState(Long meetingId) {
@ -329,22 +379,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
return Paths.get(normalizedBasePath, "android-chunks", String.valueOf(meetingId)); return Paths.get(normalizedBasePath, "android-chunks", String.valueOf(meetingId));
} }
private String buildStoredChunkFileName(Integer chunkIndex, String originalFileName) { private Path resolveChunkFilePath(Long meetingId, Integer chunkIndex, String originalFileName) {
String normalizedSourceName = normalizeChunkSourceFileName(originalFileName); return sessionDir(meetingId)
int extensionIndex = normalizedSourceName.lastIndexOf('.'); .resolve(CHUNK_ROOT_DIR)
String extension = extensionIndex >= 0 ? normalizedSourceName.substring(extensionIndex) : ""; .resolve("chunk-" + chunkIndex)
String safeExtension = extension.isBlank() ? ".bin" : extension; .resolve(originalFileName);
return "chunk-" + (chunkIndex == null ? "unknown" : chunkIndex) + safeExtension;
} }
private String buildMergedOriginalFilename(AndroidChunkUploadSessionState state, Path mergedFile) { private String resolveMergedOriginalFilename(AndroidChunkUploadSessionState state, List<Path> chunkPaths, Path mergedFile) {
if (state.getFileName() != null && !state.getFileName().isBlank()) { if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) {
String normalizedSourceName = normalizeChunkSourceFileName(state.getFileName()); return state.getFileName();
int extensionIndex = normalizedSourceName.lastIndexOf('.'); }
if (extensionIndex >= 0) { if (chunkPaths != null && !chunkPaths.isEmpty() && chunkPaths.get(0).getFileName() != null) {
return "merged" + normalizedSourceName.substring(extensionIndex); return chunkPaths.get(0).getFileName().toString();
}
return normalizedSourceName;
} }
return mergedFile == null || mergedFile.getFileName() == null ? "meeting-audio.bin" : mergedFile.getFileName().toString(); return mergedFile == null || mergedFile.getFileName() == null ? "meeting-audio.bin" : mergedFile.getFileName().toString();
} }
@ -353,41 +400,56 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (chunkPath == null || chunkPath.getFileName() == null) { if (chunkPath == null || chunkPath.getFileName() == null) {
return ".bin"; return ".bin";
} }
String fileName = normalizeChunkSourceFileName(chunkPath.getFileName().toString()); String fileName = chunkPath.getFileName().toString();
int extensionIndex = fileName.lastIndexOf('.'); int extensionIndex = fileName.lastIndexOf('.');
return extensionIndex >= 0 ? fileName.substring(extensionIndex) : ".bin"; return extensionIndex >= 0 ? fileName.substring(extensionIndex) : ".bin";
} }
private String resolveMergedExtension(AndroidChunkUploadSessionState state, List<Path> chunkPaths) { private String resolveMergedExtension(AndroidChunkUploadSessionState state, List<Path> chunkPaths) {
if (chunkPaths != null && !chunkPaths.isEmpty()) { if (chunkPaths != null && !chunkPaths.isEmpty()) {
return resolveMergedExtension(chunkPaths.get(0)); return resolveMergedExtension(chunkPaths.get(0));
} }
if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) { if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) {
int extensionIndex = state.getFileName().lastIndexOf('.'); int extensionIndex = state.getFileName().lastIndexOf('.');
if (extensionIndex >= 0) { if (extensionIndex >= 0) {
return state.getFileName().substring(extensionIndex); return state.getFileName().substring(extensionIndex);
} }
} }
return ".bin"; return ".bin";
} }
private String normalizeChunkSourceFileName(String fileName) { private boolean isPendingChunkFile(Path path) {
if (fileName == null) { return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending");
return ""; }
private String resolveOriginalFileName(String originalFileName) {
if (originalFileName == null || originalFileName.trim().isEmpty()) {
throw new RuntimeException("chunk_file原始文件名不能为空");
} }
String normalized = Paths.get(fileName.trim()).getFileName().toString(); if (originalFileName.contains("/") || originalFileName.contains("\\")) {
if (normalized.endsWith(".pending")) { throw new RuntimeException("chunk_file文件名不合法");
return normalized.substring(0, normalized.length() - ".pending".length()); }
return originalFileName;
}
private void clearChunkDirectory(Path chunkDir) throws IOException {
if (chunkDir == null || !Files.exists(chunkDir)) {
return;
}
try (var paths = Files.walk(chunkDir)) {
paths.sorted((left, right) -> right.compareTo(left)).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
});
} }
return normalized;
} }
private void writeConcatListFile(Path concatList, List<Path> chunkPaths) throws IOException { private void writeConcatListFile(Path concatList, List<Path> chunkPaths) throws IOException {
List<String> lines = new ArrayList<>(chunkPaths.size()); List<String> lines = new ArrayList<>(chunkPaths.size());
for (Path chunkPath : chunkPaths) { for (Path chunkPath : chunkPaths) {
if (chunkPath.getFileName() != null && chunkPath.getFileName().toString().endsWith(".pending")) {
continue;
}
String normalizedPath = chunkPath.toAbsolutePath().toString().replace("'", "'\\''"); String normalizedPath = chunkPath.toAbsolutePath().toString().replace("'", "'\\''");
lines.add("file '" + normalizedPath + "'"); lines.add("file '" + normalizedPath + "'");
} }
@ -424,46 +486,20 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (process.exitValue() != 0) { if (process.exitValue() != 0) {
throw new IOException("音频分片合并失败: " + new String(output, StandardCharsets.UTF_8)); throw new IOException("音频分片合并失败: " + new String(output, StandardCharsets.UTF_8));
} }
if (!Files.exists(mergedOutput)) { if (!Files.exists(mergedOutput)) {
throw new IOException("音频分片合并结果为空"); throw new IOException("音频分片合并结果为空");
} }
} }
private void deleteQuietly(Path path) {
if (path == null) {
return;
}
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private LoginUser toLoginUser(AndroidAuthContext authContext) {
if (authContext == null || authContext.isAnonymous() || authContext.getUserId() == null || authContext.getTenantId() == null) {
throw new RuntimeException("安卓用户未登录或认证无效");
}
LoginUser loginUser = new LoginUser(
authContext.getUserId(),
authContext.getTenantId(),
authContext.getUsername(),
authContext.getPlatformAdmin(),
authContext.getTenantAdmin(),
authContext.getPermissions()
);
loginUser.setDisplayName(authContext.getDisplayName());
return loginUser;
}
private static final class LocalMultipartFile implements MultipartFile { private static final class LocalMultipartFile implements MultipartFile {
private final String originalFilename; private final String originalFilename;
private final String contentType; private final String contentType;
private final byte[] bytes; private final Path filePath;
private LocalMultipartFile(String originalFilename, String contentType, byte[] bytes) { private LocalMultipartFile(String originalFilename, String contentType, Path filePath) {
this.originalFilename = originalFilename; this.originalFilename = originalFilename;
this.contentType = contentType; this.contentType = contentType;
this.bytes = bytes == null ? new byte[0] : bytes; this.filePath = filePath;
} }
@Override @Override
@ -483,27 +519,35 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
@Override @Override
public boolean isEmpty() { public boolean isEmpty() {
return bytes.length == 0; try {
return Files.size(filePath) == 0;
} catch (IOException ex) {
return true;
}
} }
@Override @Override
public long getSize() { public long getSize() {
return bytes.length; try {
return Files.size(filePath);
} catch (IOException ex) {
return 0;
}
} }
@Override @Override
public byte[] getBytes() { public byte[] getBytes() throws IOException {
return bytes; return Files.readAllBytes(filePath);
} }
@Override @Override
public InputStream getInputStream() { public InputStream getInputStream() throws IOException {
return new ByteArrayInputStream(bytes); return Files.newInputStream(filePath);
} }
@Override @Override
public void transferTo(java.io.File dest) throws IOException, IllegalStateException { public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
Files.write(dest.toPath(), bytes); Files.copy(filePath, dest.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
} }
} }
} }

View File

@ -44,6 +44,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Service @Service
@ -289,6 +290,7 @@ public class LegacyMeetingAdapterServiceImpl implements LegacyMeetingAdapterServ
String relocatedUrl = meetingDomainSupport.relocateAudioUrl(meetingId, stagingUrl); String relocatedUrl = meetingDomainSupport.relocateAudioUrl(meetingId, stagingUrl);
taskSecurityContextRunner.runAsTenantUser(meeting.getTenantId(), meeting.getCreatorId(), () -> { taskSecurityContextRunner.runAsTenantUser(meeting.getTenantId(), meeting.getCreatorId(), () -> {
meetingDomainSupport.applyMeetingAudioMetadata(meeting, relocatedUrl); meetingDomainSupport.applyMeetingAudioMetadata(meeting, relocatedUrl);
meetingDomainSupport.prewarmPlaybackAudioAfterCommit(relocatedUrl);
meeting.setSummaryModelId(profile.getResolvedSummaryModelId()); meeting.setSummaryModelId(profile.getResolvedSummaryModelId());
meeting.setPromptId(profile.getResolvedPromptId()); meeting.setPromptId(profile.getResolvedPromptId());
meeting.setAudioSaveStatus(RealtimeMeetingAudioStorageService.STATUS_SUCCESS); meeting.setAudioSaveStatus(RealtimeMeetingAudioStorageService.STATUS_SUCCESS);

View File

@ -28,14 +28,14 @@ public interface MeetingCommandService {
void deleteMeeting(Long id); void deleteMeeting(Long id);
void appendRealtimeTranscripts(Long meetingId, List<RealtimeTranscriptItemDTO> items);
void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult); void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult);
void completeRealtimeMeeting(Long meetingId, String audioUrl, boolean overwriteAudio); void completeRealtimeMeeting(Long meetingId, String audioUrl, boolean overwriteAudio);
void finishOfflineMeeting(Long meetingId, String finishStage); void finishOfflineMeeting(Long meetingId, String finishStage);
void failOfflineTranscription(Long meetingId, String failureMessage);
void updateSpeakerInfo(Long meetingId, String speakerId, String newName, String label); void updateSpeakerInfo(Long meetingId, String speakerId, String newName, String label);
void updateMeetingTranscript(UpdateMeetingTranscriptCommand command); void updateMeetingTranscript(UpdateMeetingTranscriptCommand command);

View File

@ -16,7 +16,10 @@ public interface MeetingQueryService {
MeetingVO getDetail(Long id); MeetingVO getDetail(Long id);
MeetingVO getDetailIgnoreTenant(Long id); default MeetingVO getDetailIgnoreTenant(Long id){
return getDetailIgnoreTenant(id,true);
};
MeetingVO getDetailIgnoreTenant(Long id,Boolean includeAudio);
List<MeetingTranscriptVO> getTranscripts(Long meetingId); List<MeetingTranscriptVO> getTranscripts(Long meetingId);

View File

@ -11,6 +11,8 @@ public interface RealtimeMeetingSessionStateService {
void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig); void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig);
void rememberSpeakerContext(Long meetingId, String speakerContextId);
void assertCanOpenSession(Long meetingId); void assertCanOpenSession(Long meetingId);
boolean activate(Long meetingId, String connectionId); boolean activate(Long meetingId, String connectionId);
@ -25,6 +27,8 @@ public interface RealtimeMeetingSessionStateService {
void refreshAfterTranscript(Long meetingId); void refreshAfterTranscript(Long meetingId);
void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount);
boolean markCompletingIfResumeExpired(Long meetingId); boolean markCompletingIfResumeExpired(Long meetingId);
void expireEmptySession(Long meetingId); void expireEmptySession(Long meetingId);

View File

@ -11,6 +11,7 @@ import com.imeeting.dto.biz.AiLocalProfileVO;
import com.imeeting.dto.biz.AiModelVO; import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.entity.biz.AsrModel; import com.imeeting.entity.biz.AsrModel;
import com.imeeting.entity.biz.LlmModel; import com.imeeting.entity.biz.LlmModel;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.mapper.biz.AsrModelMapper; import com.imeeting.mapper.biz.AsrModelMapper;
import com.imeeting.mapper.biz.LlmModelMapper; import com.imeeting.mapper.biz.LlmModelMapper;
import com.imeeting.service.biz.AiModelService; import com.imeeting.service.biz.AiModelService;
@ -44,6 +45,10 @@ public class AiModelServiceImpl implements AiModelService {
private static final String TYPE_ASR = "ASR"; private static final String TYPE_ASR = "ASR";
private static final String TYPE_LLM = "LLM"; private static final String TYPE_LLM = "LLM";
private static final String TENCENT_PROVIDER = "tencent";
private static final String MEDIA_TENCENT_APP_ID = "tencentAppId";
private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId";
private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey";
private static final int DEFAULT_SORT_ORDER = 0; private static final int DEFAULT_SORT_ORDER = 0;
private static final String DEFAULT_LLM_API_PATH = "/v1/chat/completions"; private static final String DEFAULT_LLM_API_PATH = "/v1/chat/completions";
private static final String DEFAULT_ANTHROPIC_API_PATH = "/messages"; private static final String DEFAULT_ANTHROPIC_API_PATH = "/messages";
@ -169,7 +174,7 @@ public class AiModelServiceImpl implements AiModelService {
if (resolvedBaseUrl == null || resolvedBaseUrl.isBlank()) { if (resolvedBaseUrl == null || resolvedBaseUrl.isBlank()) {
return Collections.emptyList(); return Collections.emptyList();
} }
if ("custom".equals(providerKey)) { if (ModelProviderEnum.LOCAL.getCode().equals(providerKey)) {
return fetchLocalProfile(resolvedBaseUrl, apiKey).getAsrModels(); return fetchLocalProfile(resolvedBaseUrl, apiKey).getAsrModels();
} }
String targetUrl = resolveModelListUrl(providerKey, resolvedBaseUrl, apiKey); String targetUrl = resolveModelListUrl(providerKey, resolvedBaseUrl, apiKey);
@ -464,7 +469,7 @@ public class AiModelServiceImpl implements AiModelService {
} }
private String resolveModelListUrl(String providerKey, String baseUrl, String apiKey) { private String resolveModelListUrl(String providerKey, String baseUrl, String apiKey) {
if ("Custom".equalsIgnoreCase(providerKey)) { if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(providerKey)) {
return baseUrl+"/api/asrconfig"; return baseUrl+"/api/asrconfig";
} }
if ("gemini".equals(providerKey) || "google".equals(providerKey)) { if ("gemini".equals(providerKey) || "google".equals(providerKey)) {
@ -740,6 +745,7 @@ public class AiModelServiceImpl implements AiModelService {
if (Integer.valueOf(1).equals(dto.getIsDefault()) && !Integer.valueOf(1).equals(dto.getStatus())) { if (Integer.valueOf(1).equals(dto.getIsDefault()) && !Integer.valueOf(1).equals(dto.getStatus())) {
throw new RuntimeException("默认模型必须为启用状态"); throw new RuntimeException("默认模型必须为启用状态");
} }
validateTencentAsrConfig(dto);
// if ("custom".equals(normalizeProvider(dto.getProvider()))) { // if ("custom".equals(normalizeProvider(dto.getProvider()))) {
// if (TYPE_ASR.equals(normalizeType(dto.getModelType()))) { // if (TYPE_ASR.equals(normalizeType(dto.getModelType()))) {
// Map<String, Object> mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig(); // Map<String, Object> mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig();
@ -832,13 +838,20 @@ public class AiModelServiceImpl implements AiModelService {
} }
private void pushAsrConfig(AsrModel entity) { private void pushAsrConfig(AsrModel entity) {
if ("custom".equals(normalizeProvider(entity.getProvider()))) { String provider = normalizeProvider(entity.getProvider());
if (ModelProviderEnum.LOCAL.getCode().equals(provider)) {
if (entity.getApiKey() == null || entity.getApiKey().isBlank()) { if (entity.getApiKey() == null || entity.getApiKey().isBlank()) {
log.info("Skip syncing local ASR profile because apiKey is blank, modelName={}", entity.getModelName()); log.info("Skip syncing local ASR profile because apiKey is blank, modelName={}", entity.getModelName());
return; return;
} }
updateLocalProfile(entity); updateLocalProfile(entity);
return; return;
}
if ("custom".equals(provider)) {
return;
}
if (TENCENT_PROVIDER.equals(provider)) {
return;
} }
if (entity.getBaseUrl() == null || entity.getBaseUrl().isBlank()) { if (entity.getBaseUrl() == null || entity.getBaseUrl().isBlank()) {
throw new RuntimeException("ASR 模型必须配置 baseUrl"); throw new RuntimeException("ASR 模型必须配置 baseUrl");
@ -934,6 +947,28 @@ public class AiModelServiceImpl implements AiModelService {
return text.isEmpty() ? null : text; return text.isEmpty() ? null : text;
} }
private void validateTencentAsrConfig(AiModelDTO dto) {
if (!TYPE_ASR.equals(normalizeType(dto.getModelType()))) {
return;
}
if (!TENCENT_PROVIDER.equals(normalizeProvider(dto.getProvider()))) {
return;
}
Map<String, Object> mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig();
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_APP_ID)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentAppId");
}
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_ID)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretId");
}
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_KEY)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey");
}
if (dto.getModelCode() == null || dto.getModelCode().isBlank()) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 modelCode");
}
}
private BigDecimal readConfigDecimal(Object value) { private BigDecimal readConfigDecimal(Object value) {
if (value == null) { if (value == null) {
return null; return null;

View File

@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.common.SysParamKeys;
import com.imeeting.common.MeetingProgressStage; import com.imeeting.common.MeetingProgressStage;
import com.imeeting.dto.biz.AiModelVO; import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.dto.biz.MeetingSummarySource; import com.imeeting.dto.biz.MeetingSummarySource;
@ -66,6 +67,8 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
private static final Duration ASR_SUBMIT_REQUEST_TIMEOUT = Duration.ofSeconds(30); private static final Duration ASR_SUBMIT_REQUEST_TIMEOUT = Duration.ofSeconds(30);
private static final Duration ASR_QUERY_REQUEST_TIMEOUT = Duration.ofSeconds(30); private static final Duration ASR_QUERY_REQUEST_TIMEOUT = Duration.ofSeconds(30);
private static final String DISPATCH_MODE_PARALLEL = "PARALLEL";
private static final String DISPATCH_MODE_SERIAL = "SERIAL";
private final MeetingMapper meetingMapper; private final MeetingMapper meetingMapper;
private final MeetingTranscriptMapper transcriptMapper; private final MeetingTranscriptMapper transcriptMapper;
@ -278,6 +281,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
failPendingSummaryTask(sumTask, "没有可用于总结的转录内容"); failPendingSummaryTask(sumTask, "没有可用于总结的转录内容");
updateMeetingStatus(meetingId, 4); updateMeetingStatus(meetingId, 4);
updateProgress(meetingId, -1, "未识别到可用于总结的转录内容", 0); updateProgress(meetingId, -1, "未识别到可用于总结的转录内容", 0);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.FAILED_TRANSCRIBING.getCode());
return; return;
} }
if (!asrText.isBlank()) { if (!asrText.isBlank()) {
@ -287,8 +291,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
sumTask == null ? null : sumTask.getId()); sumTask == null ? null : sumTask.getId());
meetingProgressService.markStage(meetingId, asrTask, 1, MeetingProgressStage.ASR_COMPLETED, 80, "转写完成,准备生成总结", 0); meetingProgressService.markStage(meetingId, asrTask, 1, MeetingProgressStage.ASR_COMPLETED, 80, "转写完成,准备生成总结", 0);
scheduleQueuedAsrTasks(); scheduleQueuedAsrTasks();
self.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); dispatchPostAsrTasks(meeting, chapterTask, sumTask);
self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
return; return;
} }
if (sumTask != null && canExecuteTask(sumTask)) { if (sumTask != null && canExecuteTask(sumTask)) {
@ -342,8 +345,20 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
return; return;
} }
executeChapterFlow(meeting, chapterTask); executeChapterFlow(meeting, chapterTask);
if (shouldRunSummaryAfterChapter(meeting, chapterTask)) {
AiTask summaryTask = findLatestTask(meetingId, "SUMMARY");
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
return;
}
}
reconcileMeetingStatus(meetingId); reconcileMeetingStatus(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.COMPLETED.getCode()); androidMeetingPushService.pushMeetingStatusChanged(
meetingId,
MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)
? UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode()
: UnifiedMeetingStatusStage.COMPLETED.getCode()
);
log.info("[CHAPTER-FLOW] 章节任务流程结束: meetingId={}, chapterTaskId={}, costMs={}", log.info("[CHAPTER-FLOW] 章节任务流程结束: meetingId={}, chapterTaskId={}, costMs={}",
meetingId, chapterTask.getId(), System.currentTimeMillis() - startMillis); meetingId, chapterTask.getId(), System.currentTimeMillis() - startMillis);
} }
@ -1106,7 +1121,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
meetingPointsService.recordSummarySuccessCharge(meeting, taskRecord); meetingPointsService.recordSummarySuccessCharge(meeting, taskRecord);
AiTask latestChapterTask = findLatestTask(meeting.getId(), "CHAPTER"); AiTask latestChapterTask = findLatestTask(meeting.getId(), "CHAPTER");
if (latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus())) { if (!resolveAiCatalogEnabled() ||(latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus()))) {
updateProgress(meeting.getId(), 100, "全流程分析完成", 0); updateProgress(meeting.getId(), 100, "全流程分析完成", 0);
} else { } else {
updateProgress(meeting.getId(), 95, "总结生成完成,等待 AI 目录完成...", 0); updateProgress(meeting.getId(), 95, "总结生成完成,等待 AI 目录完成...", 0);
@ -1177,7 +1192,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
log.info("[SUMMARY-EXEC] 已获取总结锁,开始构建总结来源: meetingId={}, sumTaskId={}", log.info("[SUMMARY-EXEC] 已获取总结锁,开始构建总结来源: meetingId={}, sumTaskId={}",
meeting.getId(), sumTask == null ? null : sumTask.getId()); meeting.getId(), sumTask == null ? null : sumTask.getId());
try { try {
MeetingSummarySource summarySource = buildRawTranscriptSummarySource(meeting); MeetingSummarySource summarySource = buildSummarySourceForExecution(meeting, sumTask);
if (summarySource.getText() == null || summarySource.getText().isBlank()) { if (summarySource.getText() == null || summarySource.getText().isBlank()) {
log.warn("[SUMMARY-EXEC] 无转录内容,无法生成总结: meetingId={}, sumTaskId={}", log.warn("[SUMMARY-EXEC] 无转录内容,无法生成总结: meetingId={}, sumTaskId={}",
meeting.getId(), sumTask == null ? null : sumTask.getId()); meeting.getId(), sumTask == null ? null : sumTask.getId());
@ -1207,6 +1222,85 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
.build(); .build();
} }
private MeetingSummarySource buildSummarySourceForExecution(Meeting meeting, AiTask sumTask) {
if (shouldUseChapterBackedSummarySource()) {
return meetingTranscriptChapterService.resolveSummarySource(meeting, sumTask);
}
return buildRawTranscriptSummarySource(meeting);
}
private void dispatchPostAsrTasks(Meeting meeting, AiTask chapterTask, AiTask summaryTask) {
if (meeting == null) {
return;
}
if (!resolveAiCatalogEnabled()) {
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
return;
}
if (isSerialDispatchMode()) {
if (chapterTask != null && canExecuteTask(chapterTask)) {
self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
} else {
log.warn("[ASR-FLOW] 串行模式下缺少可执行章节任务,跳过总结派发: meetingId={}, chapterTaskId={}, chapterStatus={}",
meeting.getId(),
chapterTask == null ? null : chapterTask.getId(),
chapterTask == null ? null : chapterTask.getStatus());
}
return;
}
if (chapterTask != null && canExecuteTask(chapterTask)) {
self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
}
private boolean shouldRunSummaryAfterChapter(Meeting meeting, AiTask chapterTask) {
return meeting != null
&& resolveAiCatalogEnabled()
&& isSerialDispatchMode()
&& isTaskCompleted(chapterTask)
&& !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED);
}
private boolean shouldUseChapterBackedSummarySource() {
return resolveAiCatalogEnabled() && isSerialDispatchMode();
}
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private boolean isSerialDispatchMode() {
return DISPATCH_MODE_SERIAL.equals(resolveSummaryDispatchMode());
}
private String resolveSummaryDispatchMode() {
if (sysParamService == null) {
return DISPATCH_MODE_PARALLEL;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, DISPATCH_MODE_PARALLEL);
if (rawValue == null || rawValue.isBlank()) {
return DISPATCH_MODE_PARALLEL;
}
String normalized = rawValue.trim().toUpperCase(Locale.ROOT);
return DISPATCH_MODE_SERIAL.equals(normalized) ? DISPATCH_MODE_SERIAL : DISPATCH_MODE_PARALLEL;
}
private AiTask findLatestTask(Long meetingId, String taskType) { private AiTask findLatestTask(Long meetingId, String taskType) {
return this.getOne(new LambdaQueryWrapper<AiTask>() return this.getOne(new LambdaQueryWrapper<AiTask>()
.eq(AiTask::getMeetingId, meetingId) .eq(AiTask::getMeetingId, meetingId)
@ -1275,7 +1369,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
updateMeetingStatus(meetingId, 4); updateMeetingStatus(meetingId, 4);
return; return;
} }
if (isTaskCompleted(chapterTask) && isTaskCompleted(summaryTask)) { if ( isTaskCompleted(summaryTask) && (!resolveAiCatalogEnabled() || isTaskCompleted(chapterTask))) {
updateMeetingStatus(meetingId, 3); updateMeetingStatus(meetingId, 3);
return; return;
} }
@ -1488,6 +1582,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
if ("SUMMARY".equals(task.getTaskType())) { if ("SUMMARY".equals(task.getTaskType())) {
meetingPointsService.markSummaryChargeFailed(task.getId(), error); meetingPointsService.markSummaryChargeFailed(task.getId(), error);
} }
androidMeetingPushService.pushMeetingStatusChanged(task.getMeetingId(), UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode());
} }
private String buildAsrFailureMessage(Exception ex) { private String buildAsrFailureMessage(Exception ex) {

View File

@ -62,6 +62,7 @@ public class MeetingAudioUploadSupport {
Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
} }
validateStoredAudio(targetPath, extension); validateStoredAudio(targetPath, extension);
} catch (Exception ex) { } catch (Exception ex) {
Files.deleteIfExists(targetPath); Files.deleteIfExists(targetPath);
throw ex; throw ex;
@ -69,6 +70,55 @@ public class MeetingAudioUploadSupport {
return buildStagingAudioToken(storedFileName); return buildStagingAudioToken(storedFileName);
} }
public String storeUploadedAudioFromPath(Path sourceFile, String originalFilename) throws IOException {
if (sourceFile == null || !Files.exists(sourceFile)) {
throw new RuntimeException("音频文件不能为空");
}
long fileSize = Files.size(sourceFile);
long maxUploadSizeMb = resolveMaxUploadSizeMb();
long maxUploadSizeBytes = maxUploadSizeMb * 1024 * 1024;
if (fileSize > maxUploadSizeBytes) {
throw new RuntimeException("音频文件大小不能超过 " + maxUploadSizeMb + "MB");
}
String extension = resolveExtension(originalFilename);
validateFileHeaderFromPath(sourceFile, extension);
Path stagingDir = resolveStagingAudioDirectory(uploadPath);
Files.createDirectories(stagingDir);
String storedFileName = UUID.randomUUID() + "." + extension;
Path targetPath = stagingDir.resolve(storedFileName);
try {
Files.move(sourceFile, targetPath, StandardCopyOption.REPLACE_EXISTING);
validateStoredAudio(targetPath, extension);
} catch (Exception ex) {
Files.deleteIfExists(targetPath);
throw ex;
}
return buildStagingAudioToken(storedFileName);
}
private void validateFileHeaderFromPath(Path sourceFile, String extension) throws IOException {
if (Files.size(sourceFile) <= 0) {
return;
}
byte[] header;
try (InputStream inputStream = Files.newInputStream(sourceFile)) {
header = inputStream.readNBytes(HEADER_SIZE);
}
boolean valid = switch (extension) {
case "wav" -> isWav(header);
case "mp3" -> isMp3(header);
case "m4a" -> isM4a(header);
default -> false;
};
if (!valid) {
throw new RuntimeException("上传文件内容与音频格式不匹配,仅支持 mp3、wav、m4a");
}
}
public static boolean isStagingAudioToken(String audioUrl) { public static boolean isStagingAudioToken(String audioUrl) {
return StringUtils.hasText(audioUrl) && audioUrl.startsWith(STAGING_AUDIO_TOKEN_PREFIX); return StringUtils.hasText(audioUrl) && audioUrl.startsWith(STAGING_AUDIO_TOKEN_PREFIX);
} }

View File

@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.common.MeetingConstants; import com.imeeting.common.MeetingConstants;
import com.imeeting.common.RedisKeys; import com.imeeting.common.RedisKeys;
import com.imeeting.common.SysParamKeys;
import com.imeeting.dto.android.AndroidPendingMeetingDraft; import com.imeeting.dto.android.AndroidPendingMeetingDraft;
import com.imeeting.dto.biz.CreateMeetingCommand; import com.imeeting.dto.biz.CreateMeetingCommand;
import com.imeeting.dto.biz.CreateRealtimeMeetingCommand; import com.imeeting.dto.biz.CreateRealtimeMeetingCommand;
@ -27,6 +28,7 @@ import com.imeeting.entity.biz.HotWord;
import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript; import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.entity.biz.MeetingTranscriptChapterVersion; import com.imeeting.entity.biz.MeetingTranscriptChapterVersion;
import com.imeeting.enums.BusinessErrorCodeEnum;
import com.imeeting.enums.MeetingStatusEnum; import com.imeeting.enums.MeetingStatusEnum;
import com.imeeting.service.android.AndroidPendingMeetingDraftService; import com.imeeting.service.android.AndroidPendingMeetingDraftService;
import com.imeeting.service.android.AndroidPushMessageService; import com.imeeting.service.android.AndroidPushMessageService;
@ -45,6 +47,10 @@ import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.imeeting.support.redis.MeetingAsrPermitCache; import com.imeeting.support.redis.MeetingAsrPermitCache;
import com.imeeting.support.redis.MeetingLockCache; import com.imeeting.support.redis.MeetingLockCache;
import com.unisbase.common.exception.BusinessException;
import com.unisbase.common.exception.ErrorCodeEnum;
import com.imeeting.websocket.RealtimeMeetingProxyWebSocketHandler;
import com.unisbase.service.SysParamService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -77,7 +83,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
private final MeetingDomainSupport meetingDomainSupport; private final MeetingDomainSupport meetingDomainSupport;
private final MeetingRuntimeProfileResolver meetingRuntimeProfileResolver; private final MeetingRuntimeProfileResolver meetingRuntimeProfileResolver;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService; private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService;
private final RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler;
private final MeetingProgressService meetingProgressService; private final MeetingProgressService meetingProgressService;
private final MeetingPointsService meetingPointsService; private final MeetingPointsService meetingPointsService;
private final MeetingSummaryPromptAssembler meetingSummaryPromptAssembler; private final MeetingSummaryPromptAssembler meetingSummaryPromptAssembler;
@ -87,6 +94,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
private final AndroidPendingMeetingDraftService androidPendingMeetingDraftService; private final AndroidPendingMeetingDraftService androidPendingMeetingDraftService;
private final MeetingLockCache meetingLockCache; private final MeetingLockCache meetingLockCache;
private final MeetingAsrPermitCache meetingAsrPermitCache; private final MeetingAsrPermitCache meetingAsrPermitCache;
private final SysParamService sysParamService;
@Value("${imeeting.summary-orchestration.mode:INTERNAL_BUILTIN}") @Value("${imeeting.summary-orchestration.mode:INTERNAL_BUILTIN}")
private String summaryOrchestrationMode; private String summaryOrchestrationMode;
@ -103,7 +111,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
MeetingDomainSupport meetingDomainSupport, MeetingDomainSupport meetingDomainSupport,
MeetingRuntimeProfileResolver meetingRuntimeProfileResolver, MeetingRuntimeProfileResolver meetingRuntimeProfileResolver,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService, RealtimeMeetingSessionStateService realtimeMeetingSessionStateService,
RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService, RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService, RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler,
MeetingProgressService meetingProgressService, MeetingProgressService meetingProgressService,
MeetingPointsService meetingPointsService, MeetingPointsService meetingPointsService,
MeetingSummaryPromptAssembler meetingSummaryPromptAssembler, MeetingSummaryPromptAssembler meetingSummaryPromptAssembler,
@ -112,7 +120,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
AndroidPushMessageService androidPushMessageService, AndroidPushMessageService androidPushMessageService,
AndroidPendingMeetingDraftService androidPendingMeetingDraftService, AndroidPendingMeetingDraftService androidPendingMeetingDraftService,
MeetingLockCache meetingLockCache, MeetingLockCache meetingLockCache,
MeetingAsrPermitCache meetingAsrPermitCache) { MeetingAsrPermitCache meetingAsrPermitCache,
SysParamService sysParamService) {
this.meetingService = meetingService; this.meetingService = meetingService;
this.aiTaskService = aiTaskService; this.aiTaskService = aiTaskService;
this.hotWordService = hotWordService; this.hotWordService = hotWordService;
@ -124,7 +133,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
this.meetingDomainSupport = meetingDomainSupport; this.meetingDomainSupport = meetingDomainSupport;
this.meetingRuntimeProfileResolver = meetingRuntimeProfileResolver; this.meetingRuntimeProfileResolver = meetingRuntimeProfileResolver;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService; this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService;
this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService; this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService;
this.realtimeMeetingProxyWebSocketHandler = realtimeMeetingProxyWebSocketHandler;
this.meetingProgressService = meetingProgressService; this.meetingProgressService = meetingProgressService;
this.meetingPointsService = meetingPointsService; this.meetingPointsService = meetingPointsService;
this.meetingSummaryPromptAssembler = meetingSummaryPromptAssembler; this.meetingSummaryPromptAssembler = meetingSummaryPromptAssembler;
@ -134,6 +144,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
this.androidPendingMeetingDraftService = androidPendingMeetingDraftService; this.androidPendingMeetingDraftService = androidPendingMeetingDraftService;
this.meetingLockCache = meetingLockCache; this.meetingLockCache = meetingLockCache;
this.meetingAsrPermitCache = meetingAsrPermitCache; this.meetingAsrPermitCache = meetingAsrPermitCache;
this.sysParamService = sysParamService;
} }
@Override @Override
@ -176,7 +187,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
aiTaskService.save(asrTask); aiTaskService.save(asrTask);
Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId(); Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId();
meetingDomainSupport.createChapterTask( createChapterTaskIfEnabled(
meeting.getId(), meeting.getId(),
runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedSummaryModelId(),
chapterModelId, chapterModelId,
@ -228,7 +239,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
hostUserId, hostName, runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedPromptId(), summaryDetailLevel, 0); hostUserId, hostName, runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedPromptId(), summaryDetailLevel, 0);
meetingService.save(meeting); meetingService.save(meeting);
Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId(); Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId();
meetingDomainSupport.createChapterTask( createChapterTaskIfEnabled(
meeting.getId(), meeting.getId(),
runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedSummaryModelId(),
chapterModelId, chapterModelId,
@ -339,58 +350,6 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
deleteMeetingArtifactsAfterCommit(id); deleteMeetingArtifactsAfterCommit(id);
} }
@Override
@Transactional(rollbackFor = Exception.class)
public void appendRealtimeTranscripts(Long meetingId, List<RealtimeTranscriptItemDTO> items) {
if (items == null || items.isEmpty()) {
return;
}
Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.orderByDesc(MeetingTranscript::getSortOrder)
.last("LIMIT 1"))
.stream()
.findFirst()
.map(MeetingTranscript::getSortOrder)
.orElse(0);
int nextSortOrder = maxSortOrder == null ? 0 : maxSortOrder + 1;
boolean inserted = false;
for (RealtimeTranscriptItemDTO item : items) {
if (item.getContent() == null || item.getContent().isBlank()) {
continue;
}
MeetingTranscript existing = transcriptMapper.selectOne(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.eq(MeetingTranscript::getContent, item.getContent().trim())
.eq(item.getSpeakerId() != null && !item.getSpeakerId().isBlank(), MeetingTranscript::getSpeakerId, item.getSpeakerId())
.eq(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime())
.eq(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime())
.last("LIMIT 1"));
if (existing != null) {
continue;
}
MeetingTranscript transcript = new MeetingTranscript();
transcript.setMeetingId(meetingId);
transcript.setSpeakerId(meetingDomainSupport.resolveSpeakerId(item.getSpeakerId()));
transcript.setSpeakerName(meetingDomainSupport.resolveSpeakerName(item.getSpeakerId(), item.getSpeakerName()));
transcript.setContent(item.getContent().trim());
transcript.setStartTime(item.getStartTime());
transcript.setEndTime(item.getEndTime());
transcript.setSortOrder(nextSortOrder++);
transcriptMapper.insert(transcript);
inserted = true;
}
if (inserted) {
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
realtimeMeetingSessionStateService.refreshAfterTranscript(meetingId);
}
}
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult) { public void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult) {
@ -449,11 +408,11 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
if (meeting == null) { if (meeting == null) {
throw new RuntimeException("会议不存在"); throw new RuntimeException("会议不存在");
} }
realtimeMeetingProxyWebSocketHandler.closeMeetingSession(meetingId);
RealtimeMeetingSessionStatusVO currentStatus = realtimeMeetingSessionStateService.getStatus(meetingId); RealtimeMeetingSessionStatusVO currentStatus = realtimeMeetingSessionStateService.getStatus(meetingId);
if (overwriteAudio) { if (overwriteAudio) {
if (audioUrl == null || audioUrl.isBlank()) { if (audioUrl == null || audioUrl.isBlank()) {
throw new RuntimeException("overwriteAudio 为 true 时必须提供音频地址"); throw new RuntimeException("overwriteAudio=true requires audioUrl");
} }
meetingDomainSupport.applyMeetingAudioMetadata( meetingDomainSupport.applyMeetingAudioMetadata(
meeting, meeting,
@ -494,10 +453,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
realtimeMeetingSessionStateService.clear(meetingId); realtimeMeetingSessionStateService.clear(meetingId);
meeting.setStatus(MeetingStatusEnum.SUMMARIZING.getCode()); meeting.setStatus(MeetingStatusEnum.SUMMARIZING.getCode());
meetingService.updateById(meeting); meetingService.updateById(meeting);
updateMeetingProgress(meetingId, 85, "正在生成 AI 目录与总结...", 0); updateMeetingProgress(meetingId, resolveAiCatalogEnabled() ? 85 : 90, resolveAiCatalogEnabled() ? "正在生成 AI 目录与总结..." : "正在生成会议总结...", 0);
meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl()); meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl());
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); if (!resolveAiCatalogEnabled()) {
aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
} else if (isParallelDispatchMode()) {
aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
} else {
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
}
} }
@Override @Override
@ -522,6 +487,22 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
meetingService.updateById(meeting); meetingService.updateById(meeting);
} }
@Override
@Transactional(rollbackFor = Exception.class)
public void failOfflineTranscription(Long meetingId, String failureMessage) {
Meeting meeting = meetingService.getById(meetingId);
if (meeting == null) {
throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(),"会议不存在");
}
if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) {
throw new RuntimeException("会议不是离线会议");
}
meeting.setStatus(MeetingStatusEnum.FAILED.getCode());
meeting.setOfflineRecordingStatus(MeetingConstants.OFFLINE_RECORDING_UPLOAD_FINISHED);
markAudioSaveFailure(meeting, failureMessage);
meetingService.updateById(meeting);
}
private void applyRealtimeAudioFinalizeResult(Meeting meeting, RealtimeMeetingAudioStorageService.FinalizeResult result) { private void applyRealtimeAudioFinalizeResult(Meeting meeting, RealtimeMeetingAudioStorageService.FinalizeResult result) {
if (result == null) { if (result == null) {
markAudioSaveFailure(meeting, RealtimeMeetingAudioStorageService.DEFAULT_FAILURE_MESSAGE); markAudioSaveFailure(meeting, RealtimeMeetingAudioStorageService.DEFAULT_FAILURE_MESSAGE);
@ -767,6 +748,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public MeetingTranscriptChapterImportResultVO importTranscriptChapters(MeetingTranscriptChapterImportDTO command) { public MeetingTranscriptChapterImportResultVO importTranscriptChapters(MeetingTranscriptChapterImportDTO command) {
ensureExternalSummaryModeEnabled(); ensureExternalSummaryModeEnabled();
ensureAiCatalogEnabled();
if (command == null || command.getMeetingId() == null) { if (command == null || command.getMeetingId() == null) {
throw new RuntimeException("缺少会议ID无法导入章节"); throw new RuntimeException("缺少会议ID无法导入章节");
} }
@ -787,6 +769,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
.last("LIMIT 1")); .last("LIMIT 1"));
if (latestChapterTask == null) { if (latestChapterTask == null) {
ensureAiCatalogEnabled();
Long summaryModelId = resolveSummaryModelId(command, latestSummaryTask); Long summaryModelId = resolveSummaryModelId(command, latestSummaryTask);
Long chapterModelId = resolveChapterModelId(command, latestSummaryTask, summaryModelId); Long chapterModelId = resolveChapterModelId(command, latestSummaryTask, summaryModelId);
Long promptId = resolvePromptId(command, latestSummaryTask); Long promptId = resolvePromptId(command, latestSummaryTask);
@ -1120,7 +1103,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
.eq(AiTask::getTaskType, "CHAPTER") .eq(AiTask::getTaskType, "CHAPTER")
.orderByDesc(AiTask::getId) .orderByDesc(AiTask::getId)
.last("LIMIT 1")); .last("LIMIT 1"));
if (chapterTask == null) { if (resolveAiCatalogEnabled() && chapterTask == null) {
chapterTask = meetingDomainSupport.createChapterTask( chapterTask = meetingDomainSupport.createChapterTask(
meetingId, meetingId,
effectiveSummaryModelId, effectiveSummaryModelId,
@ -1129,7 +1112,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
effectiveUserPrompt, effectiveUserPrompt,
effectiveSummaryDetailLevel effectiveSummaryDetailLevel
); );
} else { } else if (resolveAiCatalogEnabled()) {
resetAiTask(chapterTask, meetingSummaryPromptAssembler.buildTaskConfig( resetAiTask(chapterTask, meetingSummaryPromptAssembler.buildTaskConfig(
effectiveSummaryModelId, effectiveSummaryModelId,
effectiveChapterModelId, effectiveChapterModelId,
@ -1185,6 +1168,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
if (!Integer.valueOf(3).equals(summaryTask.getStatus())) { if (!Integer.valueOf(3).equals(summaryTask.getStatus())) {
throw new RuntimeException("当前总结环节未失败,无需重试"); throw new RuntimeException("当前总结环节未失败,无需重试");
} }
if (resolveAiCatalogEnabled() && isSerialDispatchMode()) {
AiTask chapterTask = aiTaskService.getOne(new LambdaQueryWrapper<AiTask>()
.eq(AiTask::getMeetingId, meetingId)
.eq(AiTask::getTaskType, "CHAPTER")
.orderByDesc(AiTask::getId)
.last("LIMIT 1"));
if (chapterTask == null || !Integer.valueOf(2).equals(chapterTask.getStatus())) {
throw new RuntimeException("串行模式下缺少成功的 AI 目录产物,无法重试总结");
}
}
Long effectiveSummaryModelId = resolveMeetingSummaryModelId(meeting, summaryTask); Long effectiveSummaryModelId = resolveMeetingSummaryModelId(meeting, summaryTask);
resetAiTask(summaryTask, buildSummaryTaskConfigForRetry( resetAiTask(summaryTask, buildSummaryTaskConfigForRetry(
summaryTask, summaryTask,
@ -1207,6 +1200,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void retryChapter(Long meetingId) { public void retryChapter(Long meetingId) {
ensureAiCatalogEnabled();
Meeting meeting = meetingService.getById(meetingId); Meeting meeting = meetingService.getById(meetingId);
if (meeting == null) { if (meeting == null) {
throw new RuntimeException("会议不存在"); throw new RuntimeException("会议不存在");
@ -1248,6 +1242,25 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
dispatchChapterTaskAfterCommit(meetingId, meeting.getTenantId(), meeting.getCreatorId()); dispatchChapterTaskAfterCommit(meetingId, meeting.getTenantId(), meeting.getCreatorId());
} }
private AiTask createChapterTaskIfEnabled(Long meetingId,
Long summaryModelId,
Long chapterModelId,
Long promptId,
String userPrompt,
String summaryDetailLevel) {
if (!resolveAiCatalogEnabled()) {
return null;
}
return meetingDomainSupport.createChapterTask(
meetingId,
summaryModelId,
chapterModelId,
promptId,
userPrompt,
summaryDetailLevel
);
}
private void clearLegacyDispatchState(Long meetingId) { private void clearLegacyDispatchState(Long meetingId) {
if (meetingId == null) { if (meetingId == null) {
return; return;
@ -1586,4 +1599,45 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
} }
return MeetingConstants.SUMMARY_DETAIL_STANDARD; return MeetingConstants.SUMMARY_DETAIL_STANDARD;
} }
private void ensureAiCatalogEnabled() {
if (!resolveAiCatalogEnabled()) {
throw new RuntimeException("AI目录功能未开启");
}
}
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private boolean isSerialDispatchMode() {
return "SERIAL".equals(resolveSummaryDispatchMode());
}
private boolean isParallelDispatchMode() {
return "PARALLEL".equals(resolveSummaryDispatchMode());
}
private String resolveSummaryDispatchMode() {
if (sysParamService == null) {
return "PARALLEL";
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, "PARALLEL");
if (rawValue == null || rawValue.isBlank()) {
return "PARALLEL";
}
String normalized = rawValue.trim().toUpperCase();
return "SERIAL".equals(normalized) ? "SERIAL" : "PARALLEL";
}
} }

View File

@ -1,7 +1,9 @@
package com.imeeting.service.biz.impl; package com.imeeting.service.biz.impl;
import cn.hutool.core.date.StopWatch;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.common.MeetingConstants; import com.imeeting.common.MeetingConstants;
import com.imeeting.common.SysParamKeys;
import com.imeeting.entity.biz.AiTask; import com.imeeting.entity.biz.AiTask;
import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript; import com.imeeting.entity.biz.MeetingTranscript;
@ -13,6 +15,7 @@ import com.imeeting.service.biz.MeetingSummaryFileService;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.unisbase.entity.SysUser; import com.unisbase.entity.SysUser;
import com.unisbase.mapper.SysUserMapper; import com.unisbase.mapper.SysUserMapper;
import com.unisbase.service.SysParamService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -54,6 +57,7 @@ public class MeetingDomainSupport {
private final ApplicationEventPublisher eventPublisher; private final ApplicationEventPublisher eventPublisher;
private final MeetingSummaryFileService meetingSummaryFileService; private final MeetingSummaryFileService meetingSummaryFileService;
private final MeetingPlaybackAudioResolver meetingPlaybackAudioResolver; private final MeetingPlaybackAudioResolver meetingPlaybackAudioResolver;
private final SysParamService sysParamService;
@Value("${unisbase.app.upload-path}") @Value("${unisbase.app.upload-path}")
private String uploadPath; private String uploadPath;
@ -399,6 +403,7 @@ public class MeetingDomainSupport {
vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus()); vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus());
vo.setSummaryModelId(meeting.getSummaryModelId()); vo.setSummaryModelId(meeting.getSummaryModelId());
vo.setPromptId(meeting.getPromptId()); vo.setPromptId(meeting.getPromptId());
vo.setAiCatalogEnabled(resolveAiCatalogEnabled());
vo.setSummaryDetailLevel(normalizeSummaryDetailLevel(meeting.getSummaryDetailLevel())); vo.setSummaryDetailLevel(normalizeSummaryDetailLevel(meeting.getSummaryDetailLevel()));
vo.setAudioSaveStatus(meeting.getAudioSaveStatus()); vo.setAudioSaveStatus(meeting.getAudioSaveStatus());
vo.setAudioSaveMessage(meeting.getAudioSaveMessage()); vo.setAudioSaveMessage(meeting.getAudioSaveMessage());
@ -648,6 +653,21 @@ public class MeetingDomainSupport {
return path != null && path.contains("\\"); return path != null && path.contains("\\");
} }
public boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private Path resolvePublicAudioPath(String audioUrl) { private Path resolvePublicAudioPath(String audioUrl) {
if (audioUrl == null || audioUrl.isBlank()) { if (audioUrl == null || audioUrl.isBlank()) {
return null; return null;

View File

@ -485,7 +485,7 @@ public class MeetingPlaybackAudioResolver {
"-vn", "-vn",
"-ar", String.valueOf(BROWSER_SAMPLE_RATE), "-ar", String.valueOf(BROWSER_SAMPLE_RATE),
"-c:a", "aac", "-c:a", "aac",
"-f", "mp4", "-ac", "1",
targetPath.toString() targetPath.toString()
); );
executeCommand(command, targetPath); executeCommand(command, targetPath);

View File

@ -241,6 +241,14 @@ public class MeetingProgressServiceImpl implements MeetingProgressService {
if (latestChapter != null && Integer.valueOf(1).equals(latestChapter.getStatus())) { if (latestChapter != null && Integer.valueOf(1).equals(latestChapter.getStatus())) {
return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0); return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0);
} }
if (MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.SUMMARIZING)) {
if (latestSummary != null && Integer.valueOf(2).equals(latestSummary.getStatus())) {
return buildSnapshot(meetingId, latestSummary, meeting.getStatus(), MeetingProgressStage.SUMMARY_RUNNING, 90, "正在生成会议总结...", 0);
}
if (latestChapter != null && Integer.valueOf(0).equals(latestChapter.getStatus())) {
return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0);
}
}
AiTask latestAsr = findLatestTask(meetingId, "ASR"); AiTask latestAsr = findLatestTask(meetingId, "ASR");
if (latestAsr != null) { if (latestAsr != null) {
if (Integer.valueOf(1).equals(latestAsr.getStatus())) { if (Integer.valueOf(1).equals(latestAsr.getStatus())) {

View File

@ -1,5 +1,6 @@
package com.imeeting.service.biz.impl; package com.imeeting.service.biz.impl;
import cn.hutool.core.date.StopWatch;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.imeeting.dto.biz.MeetingSummaryPromptContextRequestDTO; import com.imeeting.dto.biz.MeetingSummaryPromptContextRequestDTO;
@ -19,6 +20,7 @@ import com.imeeting.service.biz.MeetingService;
import com.imeeting.service.biz.MeetingTranscriptChapterService; import com.imeeting.service.biz.MeetingTranscriptChapterService;
import com.unisbase.dto.PageResult; import com.unisbase.dto.PageResult;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -27,6 +29,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class MeetingQueryServiceImpl implements MeetingQueryService { public class MeetingQueryServiceImpl implements MeetingQueryService {
@ -83,10 +86,12 @@ public class MeetingQueryServiceImpl implements MeetingQueryService {
return meeting != null ? toVO(meeting, true) : null; return meeting != null ? toVO(meeting, true) : null;
} }
@Override @Override
public MeetingVO getDetailIgnoreTenant(Long id) { public MeetingVO getDetailIgnoreTenant(Long id, Boolean includeAudio) {
Meeting meeting = meetingMapper.selectByIdIgnoreTenant(id); Meeting meeting = meetingMapper.selectByIdIgnoreTenant(id);
return meeting != null ? toVO(meeting, true) : null; return meeting != null ? toVO(meeting, includeAudio) : null;
} }
@Override @Override

View File

@ -369,28 +369,29 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha
private String buildChapterSystemPrompt() { private String buildChapterSystemPrompt() {
return """ return """
JSON transcript
chapters
JSONchapters , JSON
{
chapterNo,title,summary,keywords,startTranscriptId,endTranscriptId,confidence "chapters": [
{
"chapterNo": number,
"title": string,
transcript "summary": string,
"keywords": [string],
- startTranscriptId transcriptId "startTranscriptId": number,
- startTranscriptId endTranscriptId + 1 "endTranscriptId": number,
- transcript "confidence": number
}
- transcript ]
- transcript }
-
- 1.
2. transcript
transcript . 3.
4. title
5. JSON
"""; """;
} }
@ -1017,9 +1018,9 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha
Long startTranscriptId = longValue(item.path("startTranscriptId").asText(null)); Long startTranscriptId = longValue(item.path("startTranscriptId").asText(null));
Long endTranscriptId = longValue(item.path("endTranscriptId").asText(null)); Long endTranscriptId = longValue(item.path("endTranscriptId").asText(null));
Integer chapterNo = item.path("chapterNo").isInt() ? item.path("chapterNo").asInt() : null; Integer chapterNo = item.path("chapterNo").isInt() ? item.path("chapterNo").asInt() : null;
if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) { // if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) {
throw new RuntimeException("章节模型返回了不完整的章节边界"); // throw new RuntimeException("章节模型返回了不完整的章节边界");
} // }
List<String> keywords = new ArrayList<>(); List<String> keywords = new ArrayList<>();
if (item.path("keywords").isArray()) { if (item.path("keywords").isArray()) {
for (JsonNode keyword : item.path("keywords")) { for (JsonNode keyword : item.path("keywords")) {

View File

@ -2,6 +2,7 @@ package com.imeeting.service.biz.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.common.MeetingConstants; import com.imeeting.common.MeetingConstants;
import com.imeeting.common.SysParamKeys;
import com.imeeting.dto.biz.MeetingProgressSnapshot; import com.imeeting.dto.biz.MeetingProgressSnapshot;
import com.imeeting.dto.biz.MeetingVO; import com.imeeting.dto.biz.MeetingVO;
import com.imeeting.dto.biz.UnifiedMeetingStatusStage; import com.imeeting.dto.biz.UnifiedMeetingStatusStage;
@ -17,6 +18,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptChapterVersionMapper;
import com.imeeting.mapper.biz.MeetingTranscriptMapper; import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.MeetingUnifiedStatusService; import com.imeeting.service.biz.MeetingUnifiedStatusService;
import com.imeeting.support.redis.MeetingProgressCache; import com.imeeting.support.redis.MeetingProgressCache;
import com.unisbase.service.SysParamService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -31,6 +33,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
private final MeetingTranscriptMapper meetingTranscriptMapper; private final MeetingTranscriptMapper meetingTranscriptMapper;
private final MeetingTranscriptChapterVersionMapper chapterVersionMapper; private final MeetingTranscriptChapterVersionMapper chapterVersionMapper;
private final MeetingProgressCache meetingProgressCache; private final MeetingProgressCache meetingProgressCache;
private final SysParamService sysParamService;
@Override @Override
public UnifiedMeetingStatusVO resolve(MeetingVO meeting, MeetingProgressSnapshot snapshot) { public UnifiedMeetingStatusVO resolve(MeetingVO meeting, MeetingProgressSnapshot snapshot) {
@ -87,12 +90,12 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
if (isAndroidOfflineMeetingWaitingUpload(meeting)) { if (isAndroidOfflineMeetingWaitingUpload(meeting)) {
return UnifiedMeetingStatusStage.WAITING_UPLOAD; return UnifiedMeetingStatusStage.WAITING_UPLOAD;
} }
UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot); MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot);
UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot, context);
if (stageFromSnapshot != null) { if (stageFromSnapshot != null) {
return stageFromSnapshot; return stageFromSnapshot;
} }
MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot);
if (isTranscribing(context)) { if (isTranscribing(context)) {
return UnifiedMeetingStatusStage.TRANSCRIBING; return UnifiedMeetingStatusStage.TRANSCRIBING;
} }
@ -103,7 +106,8 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return UnifiedMeetingStatusStage.INITIALIZING; return UnifiedMeetingStatusStage.INITIALIZING;
} }
private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot) { private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot,
MeetingUnifiedStageContext context) {
if (snapshot == null || snapshot.getStage() == null || snapshot.getStage().isBlank()) { if (snapshot == null || snapshot.getStage() == null || snapshot.getStage().isBlank()) {
return null; return null;
} }
@ -112,15 +116,31 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
case "completed" -> UnifiedMeetingStatusStage.COMPLETED; case "completed" -> UnifiedMeetingStatusStage.COMPLETED;
case "summary_running", "chapter_running" -> UnifiedMeetingStatusStage.SUMMARIZING; case "summary_running", "chapter_running" -> UnifiedMeetingStatusStage.SUMMARIZING;
case "asr_running", "asr_completed", "asr_submitted" -> UnifiedMeetingStatusStage.TRANSCRIBING; case "asr_running", "asr_completed", "asr_submitted" -> UnifiedMeetingStatusStage.TRANSCRIBING;
case "queued" -> UnifiedMeetingStatusStage.INITIALIZING; case "queued" -> resolveQueuedSnapshotStage(context);
default -> null; default -> null;
}; };
} }
private UnifiedMeetingStatusStage resolveQueuedSnapshotStage(MeetingUnifiedStageContext context) {
if (context == null) {
return UnifiedMeetingStatusStage.INITIALIZING;
}
if (isSummarizing(context)) {
return UnifiedMeetingStatusStage.SUMMARIZING;
}
if (isTranscribing(context)) {
return UnifiedMeetingStatusStage.TRANSCRIBING;
}
return UnifiedMeetingStatusStage.INITIALIZING;
}
private UnifiedMeetingStatusStage resolveFailedStage(MeetingVO meeting) { private UnifiedMeetingStatusStage resolveFailedStage(MeetingVO meeting) {
if (meeting == null || !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)) { if (meeting == null || !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)) {
return null; return null;
} }
if (isAndroidOfflineEmptyUploadFailure(meeting)) {
return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING;
}
AiTask asrTask = findLatestTask(meeting.getId(), "ASR"); AiTask asrTask = findLatestTask(meeting.getId(), "ASR");
if (isTaskFailed(asrTask)) { if (isTaskFailed(asrTask)) {
return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING; return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING;
@ -140,6 +160,13 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return UnifiedMeetingStatusStage.FAILED_INITIALIZING; return UnifiedMeetingStatusStage.FAILED_INITIALIZING;
} }
private boolean isAndroidOfflineEmptyUploadFailure(MeetingVO meeting) {
return meeting != null
&& MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType())
&& MeetingConstants.SOURCE_ANDROID.equalsIgnoreCase(meeting.getMeetingSource())
&& "FAILED".equalsIgnoreCase(meeting.getAudioSaveStatus());
}
private boolean isAndroidOfflineMeetingWaitingUpload(MeetingVO meeting) { private boolean isAndroidOfflineMeetingWaitingUpload(MeetingVO meeting) {
return meeting != null return meeting != null
&& MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType()) && MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType())
@ -162,6 +189,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
} }
private Integer resolvePercent(MeetingProgressSnapshot snapshot, UnifiedMeetingStatusStage stage) { private Integer resolvePercent(MeetingProgressSnapshot snapshot, UnifiedMeetingStatusStage stage) {
if (stage != null && stage.getCode().startsWith("FAILED_")) {
return -1;
}
if (snapshot != null && snapshot.getPercent() != null) { if (snapshot != null && snapshot.getPercent() != null) {
return snapshot.getPercent(); return snapshot.getPercent();
} }
@ -203,6 +233,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
if (meeting == null) { if (meeting == null) {
return "处理失败"; return "处理失败";
} }
if (meeting.getAudioSaveMessage() != null && !meeting.getAudioSaveMessage().isBlank()) {
return meeting.getAudioSaveMessage();
}
if (meeting.getLatestSummaryAttemptErrorMsg() != null && !meeting.getLatestSummaryAttemptErrorMsg().isBlank()) { if (meeting.getLatestSummaryAttemptErrorMsg() != null && !meeting.getLatestSummaryAttemptErrorMsg().isBlank()) {
return meeting.getLatestSummaryAttemptErrorMsg(); return meeting.getLatestSummaryAttemptErrorMsg();
} }
@ -222,6 +255,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
} }
private boolean canViewAiChapters(Long meetingId) { private boolean canViewAiChapters(Long meetingId) {
if (!resolveAiCatalogEnabled()) {
return false;
}
return meetingId != null && chapterVersionMapper.selectCount(new LambdaQueryWrapper<MeetingTranscriptChapterVersion>() return meetingId != null && chapterVersionMapper.selectCount(new LambdaQueryWrapper<MeetingTranscriptChapterVersion>()
.eq(MeetingTranscriptChapterVersion::getMeetingId, meetingId) .eq(MeetingTranscriptChapterVersion::getMeetingId, meetingId)
.eq(MeetingTranscriptChapterVersion::getIsCurrent, 1) .eq(MeetingTranscriptChapterVersion::getIsCurrent, 1)
@ -274,6 +310,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
vo.setSourceDeviceCode(meeting.getSourceDeviceCode()); vo.setSourceDeviceCode(meeting.getSourceDeviceCode());
vo.setSourceDeviceMode(meeting.getSourceDeviceMode()); vo.setSourceDeviceMode(meeting.getSourceDeviceMode());
vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus()); vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus());
vo.setAiCatalogEnabled(resolveAiCatalogEnabled());
vo.setSummaryDetailLevel(meeting.getSummaryDetailLevel()); vo.setSummaryDetailLevel(meeting.getSummaryDetailLevel());
vo.setAudioSaveStatus(meeting.getAudioSaveStatus()); vo.setAudioSaveStatus(meeting.getAudioSaveStatus());
vo.setAudioSaveMessage(meeting.getAudioSaveMessage()); vo.setAudioSaveMessage(meeting.getAudioSaveMessage());
@ -284,6 +321,21 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return vo; return vo;
} }
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private record MeetingUnifiedStageContext(AiTask asrTask, private record MeetingUnifiedStageContext(AiTask asrTask,
AiTask chapterTask, AiTask chapterTask,
AiTask summaryTask, AiTask summaryTask,

View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.dto.biz.RealtimeMeetingResumeConfig; import com.imeeting.dto.biz.RealtimeMeetingResumeConfig;
import com.imeeting.dto.biz.RealtimeMeetingSessionState; import com.imeeting.dto.biz.RealtimeMeetingSessionState;
import com.imeeting.dto.biz.RealtimeMeetingSessionStatusVO; import com.imeeting.dto.biz.RealtimeMeetingSessionStatusVO;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript; import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.enums.MeetingStatusEnum; import com.imeeting.enums.MeetingStatusEnum;
@ -12,6 +13,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.support.redis.MeetingLockCache; import com.imeeting.support.redis.MeetingLockCache;
import com.imeeting.support.redis.RealtimeMeetingSessionCache; import com.imeeting.support.redis.RealtimeMeetingSessionCache;
import com.imeeting.support.redis.RealtimeMeetingTranscriptCache;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -31,6 +33,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
private final MeetingLockCache meetingLockCache; private final MeetingLockCache meetingLockCache;
private final MeetingTranscriptMapper transcriptMapper; private final MeetingTranscriptMapper transcriptMapper;
private final MeetingMapper meetingMapper; private final MeetingMapper meetingMapper;
private final RealtimeMeetingTranscriptCache realtimeMeetingTranscriptCache;
@Value("${imeeting.realtime.resume-window-minutes:30}") @Value("${imeeting.realtime.resume-window-minutes:30}")
private String resumeWindowMinutesValue; private String resumeWindowMinutesValue;
@ -49,8 +52,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
next.setTenantId(tenantId); next.setTenantId(tenantId);
next.setUserId(userId); next.setUserId(userId);
next.setStatus("IDLE"); next.setStatus("IDLE");
next.setHasTranscript(countTranscripts(meetingId) > 0); long transcriptCount = countCapturedTranscripts(meetingId);
next.setTranscriptCountSnapshot(countTranscripts(meetingId)); next.setHasTranscript(transcriptCount > 0);
next.setTranscriptCountSnapshot(transcriptCount);
next.setUpdatedAt(System.currentTimeMillis()); next.setUpdatedAt(System.currentTimeMillis());
writeState(next); writeState(next);
} }
@ -59,6 +63,22 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
public void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig) { public void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig) {
RealtimeMeetingSessionState state = getOrCreateState(meetingId); RealtimeMeetingSessionState state = getOrCreateState(meetingId);
state.setResumeConfig(resumeConfig); state.setResumeConfig(resumeConfig);
state.setUpdatedAt(System.currentTimeMillis());
writeState(state);
}
@Override
public void rememberSpeakerContext(Long meetingId, String speakerContextId) {
if (meetingId == null || speakerContextId == null || speakerContextId.isBlank()) {
return;
}
RealtimeMeetingSessionState state = getOrCreateState(meetingId);
RealtimeMeetingResumeConfig resumeConfig = state.getResumeConfig();
if (resumeConfig == null) {
resumeConfig = new RealtimeMeetingResumeConfig();
state.setResumeConfig(resumeConfig);
}
resumeConfig.setSpeakerContextId(speakerContextId.trim());
state.setUpdatedAt(System.currentTimeMillis()); state.setUpdatedAt(System.currentTimeMillis());
writeState(state); writeState(state);
} }
@ -72,16 +92,16 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
String currentStatus = status.getStatus(); String currentStatus = status.getStatus();
if ("COMPLETING".equals(currentStatus)) { if ("COMPLETING".equals(currentStatus)) {
throw new RuntimeException("实时会议正在结束处理中"); throw new RuntimeException("Realtime meeting is completing");
} }
if ("COMPLETED".equals(currentStatus)) { if ("COMPLETED".equals(currentStatus)) {
throw new RuntimeException("实时会议已结束"); throw new RuntimeException("Realtime meeting is already completed");
} }
if ("ACTIVE".equals(currentStatus) || Boolean.TRUE.equals(status.getActiveConnection())) { if ("ACTIVE".equals(currentStatus) || Boolean.TRUE.equals(status.getActiveConnection())) {
throw new RuntimeException("实时会议已存在活动连接"); throw new RuntimeException("Realtime meeting already has an active connection");
} }
if ("PAUSED_RESUMABLE".equals(currentStatus) && !Boolean.TRUE.equals(status.getCanResume())) { if ("PAUSED_RESUMABLE".equals(currentStatus) && !Boolean.TRUE.equals(status.getCanResume())) {
throw new RuntimeException("实时会议恢复窗口已过期"); throw new RuntimeException("Realtime meeting resume window has expired");
} }
} }
@ -102,7 +122,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
} }
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
long transcriptCount = countTranscripts(meetingId); long transcriptCount = countCapturedTranscripts(meetingId);
state.setStatus("ACTIVE"); state.setStatus("ACTIVE");
state.setHasTranscript(transcriptCount > 0); state.setHasTranscript(transcriptCount > 0);
state.setTranscriptCountSnapshot(transcriptCount); state.setTranscriptCountSnapshot(transcriptCount);
@ -182,9 +202,13 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
@Override @Override
public void refreshAfterTranscript(Long meetingId) { public void refreshAfterTranscript(Long meetingId) {
refreshAfterTranscriptCapture(meetingId, countTranscripts(meetingId));
}
@Override
public void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount) {
RealtimeMeetingSessionState state = getOrCreateState(meetingId); RealtimeMeetingSessionState state = getOrCreateState(meetingId);
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
long transcriptCount = countTranscripts(meetingId);
state.setHasTranscript(transcriptCount > 0); state.setHasTranscript(transcriptCount > 0);
state.setTranscriptCountSnapshot(transcriptCount); state.setTranscriptCountSnapshot(transcriptCount);
@ -214,7 +238,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
return false; return false;
} }
long transcriptCount = countTranscripts(meetingId); long transcriptCount = countCapturedTranscripts(meetingId);
if (transcriptCount <= 0) { if (transcriptCount <= 0) {
clear(meetingId); clear(meetingId);
return false; return false;
@ -256,7 +280,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
} }
private RealtimeMeetingSessionStatusVO pauseState(Long meetingId, RealtimeMeetingSessionState state) { private RealtimeMeetingSessionStatusVO pauseState(Long meetingId, RealtimeMeetingSessionState state) {
long transcriptCount = countTranscripts(meetingId); long transcriptCount = countCapturedTranscripts(meetingId);
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
state.setHasTranscript(transcriptCount > 0); state.setHasTranscript(transcriptCount > 0);
@ -307,7 +331,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
} else { } else {
vo.setStatus("IDLE"); vo.setStatus("IDLE");
} }
vo.setHasTranscript(countTranscripts(meetingId) > 0); vo.setHasTranscript(countCapturedTranscripts(meetingId) > 0);
vo.setCanResume(false); vo.setCanResume(false);
vo.setRemainingSeconds(0L); vo.setRemainingSeconds(0L);
vo.setActiveConnection(false); vo.setActiveConnection(false);
@ -362,8 +386,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
RealtimeMeetingSessionState next = new RealtimeMeetingSessionState(); RealtimeMeetingSessionState next = new RealtimeMeetingSessionState();
next.setMeetingId(meetingId); next.setMeetingId(meetingId);
next.setStatus("IDLE"); next.setStatus("IDLE");
next.setHasTranscript(countTranscripts(meetingId) > 0); long transcriptCount = countCapturedTranscripts(meetingId);
next.setTranscriptCountSnapshot(countTranscripts(meetingId)); next.setHasTranscript(transcriptCount > 0);
next.setTranscriptCountSnapshot(transcriptCount);
next.setUpdatedAt(System.currentTimeMillis()); next.setUpdatedAt(System.currentTimeMillis());
return next; return next;
} }
@ -400,4 +425,21 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
return transcriptMapper.selectCount(new LambdaQueryWrapper<MeetingTranscript>() return transcriptMapper.selectCount(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)); .eq(MeetingTranscript::getMeetingId, meetingId));
} }
private long countCapturedTranscripts(Long meetingId) {
return Math.max(countTranscripts(meetingId), countCachedTranscripts(meetingId));
}
private long countCachedTranscripts(Long meetingId) {
if (meetingId == null) {
return 0L;
}
RealtimeMeetingTranscriptCacheState state = realtimeMeetingTranscriptCache.getState(meetingId);
if (state == null || state.getItems() == null || state.getItems().isEmpty()) {
return 0L;
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.count();
}
} }

View File

@ -10,12 +10,13 @@ import com.imeeting.service.biz.AiModelService;
import com.imeeting.service.biz.MeetingAccessService; import com.imeeting.service.biz.MeetingAccessService;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.biz.RealtimeMeetingSocketSessionService; import com.imeeting.service.biz.RealtimeMeetingSocketSessionService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelFactory;
import com.imeeting.support.redis.RealtimeMeetingSocketSessionCache; import com.imeeting.support.redis.RealtimeMeetingSocketSessionCache;
import com.unisbase.security.LoginUser; import com.unisbase.security.LoginUser;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
@ -30,6 +31,7 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
private final MeetingAccessService meetingAccessService; private final MeetingAccessService meetingAccessService;
private final AiModelService aiModelService; private final AiModelService aiModelService;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeAsrChannelFactory realtimeAsrChannelFactory;
@Override @Override
public RealtimeSocketSessionVO createSession(Long meetingId, Long asrModelId, String mode, String language, public RealtimeSocketSessionVO createSession(Long meetingId, Long asrModelId, String mode, String language,
@ -37,10 +39,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
Boolean enableTextRefine, Boolean saveAudio, Boolean enableTextRefine, Boolean saveAudio,
List<Map<String, Object>> hotwords, LoginUser loginUser) { List<Map<String, Object>> hotwords, LoginUser loginUser) {
if (meetingId == null) { if (meetingId == null) {
throw new RuntimeException("会议 ID 不能为空"); throw new RuntimeException("会议 ID 不能为空");
} }
if (asrModelId == null) { if (asrModelId == null) {
throw new RuntimeException("ASR 模型 ID 不能为空"); throw new RuntimeException("ASR 模型 ID 不能为空");
} }
Meeting meeting = meetingAccessService.requireMeeting(meetingId); Meeting meeting = meetingAccessService.requireMeeting(meetingId);
@ -51,12 +53,13 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
AiModelVO asrModel = aiModelService.getModelById(asrModelId, "ASR"); AiModelVO asrModel = aiModelService.getModelById(asrModelId, "ASR");
if (asrModel == null) { if (asrModel == null) {
throw new RuntimeException("ASR 模型不存在"); throw new RuntimeException("ASR 模型不存在");
} }
String targetWsUrl = resolveWsUrl(asrModel); RealtimeAsrChannel realtimeAsrChannel = realtimeAsrChannelFactory.getRequired(asrModel.getProvider());
String targetWsUrl = realtimeAsrChannel.resolveTargetWsUrl(asrModel);
if (targetWsUrl == null || targetWsUrl.isBlank()) { if (targetWsUrl == null || targetWsUrl.isBlank()) {
throw new RuntimeException("ASR 模型未配置 WebSocket 地址"); throw new RuntimeException("ASR 模型未配置 WebSocket 地址");
} }
RealtimeMeetingResumeConfig resumeConfig = new RealtimeMeetingResumeConfig(); RealtimeMeetingResumeConfig resumeConfig = new RealtimeMeetingResumeConfig();
@ -82,7 +85,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
sessionData.setUserId(loginUser.getUserId()); sessionData.setUserId(loginUser.getUserId());
sessionData.setTenantId(loginUser.getTenantId()); sessionData.setTenantId(loginUser.getTenantId());
sessionData.setAsrModelId(asrModelId); sessionData.setAsrModelId(asrModelId);
sessionData.setProvider(realtimeAsrChannelFactory.normalizeProvider(asrModel.getProvider()));
sessionData.setTargetWsUrl(targetWsUrl); sessionData.setTargetWsUrl(targetWsUrl);
sessionData.setModelCode(asrModel.getModelCode());
sessionData.setMediaConfig(asrModel.getMediaConfig());
String sessionToken = UUID.randomUUID().toString().replace("-", ""); String sessionToken = UUID.randomUUID().toString().replace("-", "");
socketSessionCache.save(sessionToken, sessionData); socketSessionCache.save(sessionToken, sessionData);
@ -91,9 +97,8 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
vo.setSessionToken(sessionToken); vo.setSessionToken(sessionToken);
vo.setPath(WS_PATH); vo.setPath(WS_PATH);
vo.setExpiresInSeconds(socketSessionCache.getSessionTtlSeconds()); vo.setExpiresInSeconds(socketSessionCache.getSessionTtlSeconds());
vo.setStartMessage(buildStartMessage( vo.setStartMessage(realtimeAsrChannel.buildStartMessage(
asrModel, asrModel,
meetingId,
mode, mode,
language, language,
useSpkId, useSpkId,
@ -110,74 +115,4 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
public RealtimeSocketSessionData getSessionData(String sessionToken) { public RealtimeSocketSessionData getSessionData(String sessionToken) {
return socketSessionCache.get(sessionToken); return socketSessionCache.get(sessionToken);
} }
private String resolveWsUrl(AiModelVO model) {
if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) {
return model.getWsUrl();
}
if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) {
return "";
}
return model.getBaseUrl()
.replaceFirst("^http://", "ws://")
.replaceFirst("^https://", "wss://");
}
private Map<String, Object> buildStartMessage(AiModelVO model, Long meetingId, String mode, String language,
Integer useSpkId, Boolean enablePunctuation, Boolean enableItn,
Boolean enableTextRefine, Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
root.put("request_id", "web_" + System.currentTimeMillis() + "_" + meetingId);
root.put("authorization", buildAuthorization(model.getApiKey()));
Map<String, Object> config = new HashMap<>();
Map<String, Object> audio = new HashMap<>();
audio.put("format", "pcm");
audio.put("sample_rate", 16000);
audio.put("channels", 1);
config.put("audio", audio);
Map<String, Object> recognition = new HashMap<>();
recognition.put("language", normalizeLanguage(language));
recognition.put("enable_punctuation", boolOrDefault(enablePunctuation, true));
recognition.put("enable_itn", boolOrDefault(enableItn, true));
recognition.put("enable_speaker", Integer.valueOf(1).equals(useSpkId));
recognition.put("enable_two_pass", !"online".equalsIgnoreCase(mode));
recognition.put("enable_text_refine", boolOrDefault(enableTextRefine, false));
recognition.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig()));
recognition.put("hotwords", hotwords == null ? List.of() : hotwords);
config.put("recognition", recognition);
config.put("model", model.getModelCode());
config.put("save_audio", boolOrDefault(saveAudio, false));
root.put("config", config);
return root;
}
private String buildAuthorization(String apiKey) {
if (apiKey == null || apiKey.isBlank()) {
return "";
}
return apiKey.startsWith("Bearer ") ? apiKey : "Bearer " + apiKey;
}
private Object readSpeakerThreshold(Map<String, Object> mediaConfig) {
if (mediaConfig == null) {
return null;
}
return mediaConfig.get("svThreshold");
}
private String normalizeLanguage(String language) {
if (language == null || language.isBlank()) {
return "auto";
}
return language.trim();
}
private boolean boolOrDefault(Boolean value, boolean defaultValue) {
return value != null ? value : defaultValue;
}
} }

View File

@ -0,0 +1,36 @@
package com.imeeting.service.realtime;
import com.imeeting.dto.biz.AiModelVO;
import java.util.List;
import java.util.Map;
public interface RealtimeAsrChannel {
boolean supports(String provider);
String resolveTargetWsUrl(AiModelVO model);
Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords);
void connect(RealtimeAsrChannelContext context) throws Exception;
void handleFrontendText(RealtimeAsrChannelContext context, String payload);
void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload);
default void onFrontendDetached(RealtimeAsrChannelContext context) {
// default no-op
}
void closeMeeting(RealtimeAsrChannelContext context);
boolean isOpen(RealtimeAsrChannelContext context);
}

View File

@ -0,0 +1,17 @@
package com.imeeting.service.realtime;
import org.springframework.web.socket.CloseStatus;
public interface RealtimeAsrChannelCallback {
void onChannelOpen(Long meetingId) throws Exception;
void sendFrontendText(Long meetingId, String payload) throws Exception;
void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception;
void sendFrontendError(Long meetingId, String code, String message);
void removeMeetingSession(Long meetingId);
void closeFrontend(Long meetingId, CloseStatus status);
}

View File

@ -0,0 +1,26 @@
package com.imeeting.service.realtime;
import lombok.Data;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Data
public class RealtimeAsrChannelContext {
private Long meetingId;
private String provider;
private String targetWsUrl;
private WebSocketSession rawSession;
private ConcurrentWebSocketSessionDecorator frontendSession;
private RealtimeAsrChannelCallback callback;
private final ConcurrentMap<String, Object> channelState = new ConcurrentHashMap<>();
private volatile ConcurrentMap<String, Object> frontendState = new ConcurrentHashMap<>();
public void bindFrontendSession(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) {
this.rawSession = rawSession;
this.frontendSession = frontendSession;
this.frontendState = new ConcurrentHashMap<>();
}
}

View File

@ -0,0 +1,29 @@
package com.imeeting.service.realtime;
import com.imeeting.enums.ModelProviderEnum;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@RequiredArgsConstructor
public class RealtimeAsrChannelFactory {
private final List<RealtimeAsrChannel> channels;
public RealtimeAsrChannel getRequired(String provider) {
String normalizedProvider = normalizeProvider(provider);
return channels.stream()
.filter(channel -> channel.supports(normalizedProvider))
.findFirst()
.orElseThrow(() -> new RuntimeException("暂不支持的实时 ASR 渠道: " + provider));
}
public String normalizeProvider(String provider) {
if (provider == null || provider.isBlank()) {
return ModelProviderEnum.LOCAL.getCode();
}
return provider.trim().toLowerCase();
}
}

View File

@ -0,0 +1,13 @@
package com.imeeting.service.realtime;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import java.util.List;
public interface RealtimeMeetingTranscriptCacheService {
void mergeUpstreamMessage(Long meetingId, String payload);
List<RealtimeMeetingTranscriptCacheItem> listOrderedItems(Long meetingId);
void clear(Long meetingId);
}

View File

@ -0,0 +1,538 @@
package com.imeeting.service.realtime.impl;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.CloseStatus;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@Slf4j
@Component
@RequiredArgsConstructor
public class LocalRealtimeAsrChannel implements RealtimeAsrChannel {
private static final String STATE_UPSTREAM_SOCKET = "upstreamSocket";
private static final String STATE_CLOSE_AFTER_END = "closeAfterEnd";
private static final String STATE_START_MESSAGE_FORWARDED = "startMessageForwarded";
private static final String STATE_UPSTREAM_SEND_CHAIN = "upstreamSendChain";
private static final String STATE_START_MESSAGE_SENT = "startMessageSent";
private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final CompletableFuture<Void> COMPLETED = CompletableFuture.completedFuture(null);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String STOP_MESSAGE = "{\"type\":\"stop\"}";
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
@Override
public boolean supports(String provider) {
return ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(provider);
}
@Override
public String resolveTargetWsUrl(AiModelVO model) {
if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) {
return model.getWsUrl();
}
if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) {
return "";
}
return model.getBaseUrl()
.replaceFirst("^http://", "ws://")
.replaceFirst("^https://", "wss://");
}
@Override
public Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
Map<String, Object> payload = new HashMap<>();
payload.put("format", "pcm");
payload.put("sample_rate", 16000);
payload.put("language", normalizeProtocolLanguage(language));
payload.put("context", "");
payload.put("enable_inverse_text_normalization", boolOrDefault(enableItn, true));
payload.put("unfixed_token_num", 3);
payload.put("silence_duration_ms", 800);
payload.put("min_partial_sec", 0.3D);
payload.put("pre_roll_ms", 240);
payload.put("max_sentence_count", 8);
payload.put("partial_holdback_chars", 2);
payload.put("enable_native_partial_stream", false);
payload.put("enable_speaker", Integer.valueOf(1).equals(useSpkId));
payload.put("match_speaker_registry", false);
payload.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig()));
payload.put("enable_realtime_longform", false);
payload.put("enable_realtime_vad_split", false);
payload.put("force_stable_segment_sec", 6);
payload.put("force_stable_min_chars", 24);
payload.put("max_segment_sec", 12);
payload.put("hotwords", hotwords == null ? List.of() : hotwords);
root.put("payload", payload);
return root;
}
@Override
public void connect(RealtimeAsrChannelContext context) throws Exception {
initializeFrontendState(context);
java.net.http.WebSocket upstreamSocket = java.net.http.HttpClient.newHttpClient()
.newWebSocketBuilder()
.buildAsync(URI.create(context.getTargetWsUrl()), new UpstreamListener(context))
.get();
context.getChannelState().put(STATE_UPSTREAM_SOCKET, upstreamSocket);
}
@Override
public void handleFrontendText(RealtimeAsrChannelContext context, String payload) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
initializeFrontendState(context);
if (looksLikeStartMessage(payload)) {
context.getFrontendState().put(STATE_START_MESSAGE_SENT, Boolean.TRUE);
if (!Boolean.TRUE.equals(context.getChannelState().get(STATE_START_MESSAGE_FORWARDED))) {
context.getChannelState().put(STATE_START_MESSAGE_FORWARDED, Boolean.TRUE);
sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text-start");
}
flushPendingAudioFrames(context, upstreamSocket);
return;
}
if (looksLikeStopMessage(payload)) {
context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE);
}
sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text");
}
@Override
public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
initializeFrontendState(context);
if (!Boolean.TRUE.equals(context.getFrontendState().get(STATE_START_MESSAGE_SENT))) {
queuePendingAudioFrame(context, payload);
return;
}
sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary");
}
@Override
public void closeMeeting(RealtimeAsrChannelContext context) {
if (context == null) {
return;
}
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE);
upstreamSocket.sendText(STOP_MESSAGE, true);
}
@Override
public boolean isOpen(RealtimeAsrChannelContext context) {
return getUpstreamSocket(context) != null;
}
public static String buildFrontendTranscriptMessage(RealtimeMeetingTranscriptCacheItem item) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
boolean isFinal = Boolean.TRUE.equals(item.getFinalResult());
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", item.getContent());
data.put("is_final", isFinal);
if (item.getSentenceId() != null) {
data.put("sentence_id", item.getSentenceId());
}
if (item.getStartTime() != null) {
data.put("start", item.getStartTime() / 1000D);
}
if (item.getEndTime() != null) {
data.put("end", item.getEndTime() / 1000D);
}
if (item.getSpeakerId() != null && !item.getSpeakerId().isBlank()) {
data.put("speaker_id", item.getSpeakerId());
}
if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) {
data.put("speaker_name", item.getSpeakerName());
}
if (item.getUserId() != null && !item.getUserId().isBlank()) {
data.put("user_id", item.getUserId());
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private Object readSpeakerThreshold(Map<String, Object> mediaConfig) {
if (mediaConfig == null) {
return null;
}
return mediaConfig.get("svThreshold");
}
private String normalizeProtocolLanguage(String language) {
if (language == null || language.isBlank()) {
return null;
}
String normalized = language.trim();
if ("auto".equalsIgnoreCase(normalized)) {
return null;
}
return normalized;
}
private boolean boolOrDefault(Boolean value, boolean defaultValue) {
return value != null ? value : defaultValue;
}
private void initializeFrontendState(RealtimeAsrChannelContext context) {
context.getFrontendState().putIfAbsent(STATE_UPSTREAM_SEND_CHAIN, COMPLETED);
context.getFrontendState().putIfAbsent(STATE_START_MESSAGE_SENT, Boolean.FALSE);
context.getFrontendState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
private java.net.http.WebSocket getUpstreamSocket(RealtimeAsrChannelContext context) {
Object value = context.getChannelState().get(STATE_UPSTREAM_SOCKET);
return value instanceof java.net.http.WebSocket socket ? socket : null;
}
@SuppressWarnings("unchecked")
private void sendUpstreamOrdered(RealtimeAsrChannelContext context,
Supplier<CompletableFuture<?>> sendAction,
String messageType) {
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
CompletableFuture<Void> chain = (CompletableFuture<Void>) frontendState.getOrDefault(STATE_UPSTREAM_SEND_CHAIN, COMPLETED);
CompletableFuture<Void> nextChain = chain
.exceptionally(ex -> null)
.thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null));
nextChain = nextChain.whenComplete((ignored, ex) -> {
if (ex != null) {
log.error("顺序发送上游消息失败meetingId={}, sessionId={}, type={}",
context.getMeetingId(), currentConnectionId(context), messageType, ex);
}
});
frontendState.put(STATE_UPSTREAM_SEND_CHAIN, nextChain);
}
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) {
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
List<byte[]> pendingFrames = (List<byte[]>) frontendState.get(STATE_PENDING_AUDIO_FRAMES);
if (pendingFrames == null) {
pendingFrames = new ArrayList<>();
frontendState.put(STATE_PENDING_AUDIO_FRAMES, pendingFrames);
}
pendingFrames.add(payload);
}
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(RealtimeAsrChannelContext context, java.net.http.WebSocket upstreamSocket) {
List<byte[]> pendingFrames;
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
pendingFrames = (List<byte[]>) frontendState.get(STATE_PENDING_AUDIO_FRAMES);
if (pendingFrames == null || pendingFrames.isEmpty()) {
return;
}
frontendState.put(STATE_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
log.info("start 后开始补发排队音频帧meetingId={}, sessionId={}, frameCount={}",
context.getMeetingId(), currentConnectionId(context), pendingFrames.size());
for (byte[] frame : pendingFrames) {
sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush");
}
}
private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId();
}
private static ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
private static boolean shouldLogBinaryFrame(int count) {
return count <= 3 || count % 25 == 0;
}
private static String summarizeText(String payload) {
if (payload == null) {
return "";
}
String normalized = payload.replaceAll("\\s+", " ").trim();
if (normalized.length() <= 240) {
return normalized;
}
return normalized.substring(0, 240) + "...";
}
private static boolean looksLikeStartMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private static boolean looksLikeStopMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"stop\"");
}
private static boolean looksLikeEndMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
try {
JsonNode root = OBJECT_MAPPER.readTree(payload);
return "end".equals(root.path("type").asText(""));
} catch (Exception ex) {
return false;
}
}
private static List<String> normalizeFrontendMessages(String upstreamPayload) {
try {
JsonNode root = OBJECT_MAPPER.readTree(upstreamPayload);
String type = root.path("type").asText("");
if (!"sentences".equals(type) && !"end".equals(type)) {
return List.of(upstreamPayload);
}
List<String> normalizedMessages = new ArrayList<>();
JsonNode sentences = root.path("sentences");
if (sentences.isArray()) {
for (JsonNode sentence : sentences) {
String text = sentence.path("sentence").asText("").trim();
if (text.isEmpty()) {
continue;
}
boolean isFinal = sentence.path("sentence_type").asInt(0) != 0 || "end".equals(type);
normalizedMessages.add(buildFrontendTranscriptMessage(sentence, text, isFinal));
}
}
if (normalizedMessages.isEmpty()) {
String fallbackText = root.path("result").path("voice_text_str").asText("").trim();
if (!fallbackText.isEmpty()) {
normalizedMessages.add(buildFrontendTranscriptMessage((JsonNode) null, fallbackText, true));
}
}
return normalizedMessages.isEmpty() ? List.of(upstreamPayload) : normalizedMessages;
} catch (Exception ex) {
return List.of(upstreamPayload);
}
}
private static String buildFrontendTranscriptMessage(JsonNode sentence, String text, boolean isFinal) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", text);
data.put("is_final", isFinal);
if (sentence != null && sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt()) {
data.put("sentence_id", sentence.get("sentence_id").asInt());
}
if (sentence != null) {
copyOptionalTimeSeconds(sentence, data, "start_time", "start");
copyOptionalTimeSeconds(sentence, data, "end_time", "end");
copyOptionalText(sentence, data, "speaker_id");
copyOptionalText(sentence, data, "speaker_name");
copyOptionalText(sentence, data, "user_id");
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private static void copyOptionalTimeSeconds(JsonNode source, ObjectNode target, String sourceFieldName, String targetFieldName) {
if (source == null) {
return;
}
if (source.has(sourceFieldName) && source.get(sourceFieldName).isNumber()) {
target.put(targetFieldName, source.get(sourceFieldName).asDouble() / 1000D);
return;
}
if (source.has(targetFieldName) && source.get(targetFieldName).isNumber()) {
target.put(targetFieldName, source.get(targetFieldName).asDouble());
}
}
private static void copyOptionalText(JsonNode source, ObjectNode target, String fieldName) {
if (source == null || !source.has(fieldName) || source.get(fieldName).isNull()) {
return;
}
String value = source.get(fieldName).asText("").trim();
if (!value.isEmpty()) {
target.put(fieldName, value);
}
}
private final class UpstreamListener implements java.net.http.WebSocket.Listener {
private final RealtimeAsrChannelContext context;
private final StringBuilder textBuffer = new StringBuilder();
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
private final AtomicInteger upstreamTextCount = new AtomicInteger();
private final AtomicInteger upstreamBinaryCount = new AtomicInteger();
private UpstreamListener(RealtimeAsrChannelContext context) {
this.context = context;
}
@Override
public void onOpen(java.net.http.WebSocket webSocket) {
context.getChannelState().put(STATE_UPSTREAM_SOCKET, webSocket);
log.info("上游 ASR websocket 已打开meetingId={}, sessionId={}, upstream={}",
context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl());
String connectionId = currentConnectionId(context);
if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) {
context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接");
webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "当前会议无法激活这条前端连接");
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接"));
return;
}
try {
context.getCallback().onChannelOpen(context.getMeetingId());
} catch (Exception ex) {
log.error("通知前端上游就绪失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
return;
}
webSocket.request(1);
}
@Override
public java.util.concurrent.CompletionStage<?> onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) {
textBuffer.append(data);
if (last) {
int count = upstreamTextCount.incrementAndGet();
String upstreamPayload = textBuffer.toString();
realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), upstreamPayload);
try {
for (String frontendPayload : normalizeFrontendMessages(upstreamPayload)) {
context.getCallback().sendFrontendText(context.getMeetingId(), frontendPayload);
}
log.info("上游 ASR 文本 -> 前端meetingId={}, sessionId={}, count={}, payload={}",
context.getMeetingId(), currentConnectionId(context), count, summarizeText(upstreamPayload));
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_CLOSE_AFTER_END))
&& looksLikeEndMessage(upstreamPayload)) {
webSocket.sendClose(CloseStatus.NORMAL.getCode(), "meeting-complete");
}
} catch (Exception ex) {
log.error("转发上游 ASR 文本失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
} finally {
textBuffer.setLength(0);
}
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) {
byte[] chunk = new byte[data.remaining()];
data.get(chunk);
binaryBuffer.writeBytes(chunk);
if (last) {
int count = upstreamBinaryCount.incrementAndGet();
try {
context.getCallback().sendFrontendBinary(context.getMeetingId(), binaryBuffer.toByteArray());
if (shouldLogBinaryFrame(count)) {
log.info("上游 ASR 二进制消息 -> 前端meetingId={}, sessionId={}, count={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), count, binaryBuffer.size());
}
} catch (Exception ex) {
log.error("转发上游 ASR 二进制消息失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
} finally {
binaryBuffer.reset();
}
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPing(java.net.http.WebSocket webSocket, ByteBuffer message) {
webSocket.sendPong(copyBuffer(message));
log.info("上游 ASR ping 已本地响应meetingId={}, sessionId={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), message.remaining());
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPong(java.net.http.WebSocket webSocket, ByteBuffer message) {
log.debug("上游 ASR pong 已本地忽略meetingId={}, sessionId={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), message.remaining());
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) {
log.info("上游 ASR websocket 已关闭meetingId={}, sessionId={}, code={}, reason={}",
context.getMeetingId(), currentConnectionId(context), statusCode, reason);
context.getChannelState().remove(STATE_UPSTREAM_SOCKET);
context.getCallback().removeMeetingSession(context.getMeetingId());
context.getCallback().sendFrontendError(context.getMeetingId(),
"REALTIME_UPSTREAM_CLOSED",
reason == null || reason.isBlank() ? "上游 ASR WebSocket 已断开" : "上游 ASR WebSocket 已断开: " + reason);
context.getCallback().closeFrontend(context.getMeetingId(), new CloseStatus(statusCode, reason));
return COMPLETED;
}
@Override
public void onError(java.net.http.WebSocket webSocket, Throwable error) {
log.error("上游 ASR websocket 异常meetingId={}, sessionId={}, upstream={}",
context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl(), error);
context.getChannelState().remove(STATE_UPSTREAM_SOCKET);
context.getCallback().removeMeetingSession(context.getMeetingId());
context.getCallback().sendFrontendError(context.getMeetingId(),
"REALTIME_UPSTREAM_ERROR",
error == null || error.getMessage() == null || error.getMessage().isBlank()
? "上游 ASR WebSocket 连接异常"
: "上游 ASR WebSocket 连接异常: " + error.getMessage());
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
}
}
}

View File

@ -0,0 +1,291 @@
package com.imeeting.service.realtime.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.MeetingTranscriptFileService;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.imeeting.support.redis.RealtimeMeetingTranscriptCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@Service
public class RealtimeMeetingTranscriptCacheServiceImpl implements RealtimeMeetingTranscriptCacheService {
private final RealtimeMeetingTranscriptCache transcriptCache;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final MeetingTranscriptMapper transcriptMapper;
private final MeetingTranscriptFileService meetingTranscriptFileService;
private final ObjectMapper objectMapper = new ObjectMapper();
private final Map<Long, Object> meetingLocks = new ConcurrentHashMap<>();
@Autowired
public RealtimeMeetingTranscriptCacheServiceImpl(RealtimeMeetingTranscriptCache transcriptCache,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService,
MeetingTranscriptMapper transcriptMapper,
MeetingTranscriptFileService meetingTranscriptFileService) {
this.transcriptCache = transcriptCache;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService;
this.transcriptMapper = transcriptMapper;
this.meetingTranscriptFileService = meetingTranscriptFileService;
}
@Override
public void mergeUpstreamMessage(Long meetingId, String payload) {
if (meetingId == null || payload == null || payload.isBlank()) {
return;
}
synchronized (lockForMeeting(meetingId)) {
RealtimeMeetingTranscriptCacheState state = getOrCreateState(meetingId);
try {
JsonNode root = objectMapper.readTree(payload);
String type = root.path("type").asText("");
if (!"sentences".equals(type) && !"end".equals(type)) {
return;
}
JsonNode sentences = root.path("sentences");
if (!sentences.isArray()) {
return;
}
for (JsonNode sentence : sentences) {
RealtimeMeetingTranscriptCacheItem item = mergeSentenceNode(state, sentence, "end".equals(type));
persistFinalSentence(meetingId, item);
}
state.setUpdatedAt(System.currentTimeMillis());
transcriptCache.saveState(state);
realtimeMeetingSessionStateService.refreshAfterTranscriptCapture(meetingId, countNonEmptyItems(state));
} catch (Exception ignored) {
// ignore malformed upstream payload
}
}
}
@Override
public List<RealtimeMeetingTranscriptCacheItem> listOrderedItems(Long meetingId) {
RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId);
if (state == null || state.getItems() == null || state.getItems().isEmpty()) {
return List.of();
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.sorted(Comparator.comparing(item -> item.getSortOrder() == null ? Integer.MAX_VALUE : item.getSortOrder()))
.toList();
}
@Override
public void clear(Long meetingId) {
transcriptCache.clear(meetingId);
}
private RealtimeMeetingTranscriptCacheItem mergeSentenceNode(RealtimeMeetingTranscriptCacheState state,
JsonNode sentence,
boolean fromEndMessage) {
String text = sentence.path("sentence").asText("").trim();
if (text.isEmpty()) {
return null;
}
Integer sentenceId = sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt()
? sentence.get("sentence_id").asInt()
: null;
String upstreamSentenceKey = readText(sentence, "sentence_key");
String sentenceKey = upstreamSentenceKey != null && !upstreamSentenceKey.isBlank()
? upstreamSentenceKey
: sentenceId == null ? "sentence-" + nextLegacySequence(state) : "sentence-" + sentenceId;
RealtimeMeetingTranscriptCacheItem item = findBySentenceKey(state, sentenceKey);
long now = System.currentTimeMillis();
if (item == null) {
item = new RealtimeMeetingTranscriptCacheItem();
item.setSentenceKey(sentenceKey);
item.setSentenceGroupKey(upstreamSentenceKey);
item.setSentenceId(sentenceId);
item.setSortOrder(nextSortOrder(state));
item.setFirstReceivedAt(now);
state.getItems().add(item);
}
item.setSentenceGroupKey(upstreamSentenceKey);
item.setSentenceType(readInteger(sentence, "sentence_type"));
item.setSpeakerId(resolveSpeakerId(sentence));
item.setSpeakerName(readText(sentence, "speaker_name"));
item.setUserId(readText(sentence, "user_id"));
item.setStartTime(readTimeMilliseconds(sentence, "start_time", "start"));
item.setEndTime(readTimeMilliseconds(sentence, "end_time", "end"));
item.setContent(text);
item.setFinalResult(fromEndMessage || Objects.equals(item.getSentenceType(), 1));
item.setUpdatedAt(now);
return item;
}
private void persistFinalSentence(Long meetingId, RealtimeMeetingTranscriptCacheItem item) {
if (meetingId == null || item == null || !Boolean.TRUE.equals(item.getFinalResult())) {
return;
}
String content = item.getContent() == null ? null : item.getContent().trim();
if (content == null || content.isBlank()) {
return;
}
String speakerId = resolveSpeakerId(item);
String speakerName = resolveSpeakerName(item);
if (item.getTranscriptId() != null) {
transcriptMapper.update(null, new LambdaUpdateWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getId, item.getTranscriptId())
.set(MeetingTranscript::getSpeakerId, speakerId)
.set(MeetingTranscript::getSpeakerName, speakerName)
.set(MeetingTranscript::getContent, content)
.set(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime())
.set(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime()));
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
return;
}
MeetingTranscript transcript = new MeetingTranscript();
transcript.setMeetingId(meetingId);
transcript.setSpeakerId(speakerId);
transcript.setSpeakerName(speakerName);
transcript.setContent(content);
transcript.setStartTime(item.getStartTime());
transcript.setEndTime(item.getEndTime());
transcript.setSortOrder(nextPersistedSortOrder(meetingId));
transcriptMapper.insert(transcript);
item.setTranscriptId(transcript.getId());
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
}
private int nextPersistedSortOrder(Long meetingId) {
Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.orderByDesc(MeetingTranscript::getSortOrder)
.last("LIMIT 1"))
.stream()
.findFirst()
.map(MeetingTranscript::getSortOrder)
.orElse(0);
return maxSortOrder == null ? 0 : maxSortOrder + 1;
}
private RealtimeMeetingTranscriptCacheItem findBySentenceKey(RealtimeMeetingTranscriptCacheState state, String sentenceKey) {
if (state.getItems() == null || state.getItems().isEmpty()) {
return null;
}
return state.getItems().stream()
.filter(item -> sentenceKey.equals(item.getSentenceKey()))
.findFirst()
.orElse(null);
}
private String resolveSpeakerId(RealtimeMeetingTranscriptCacheItem item) {
if (item == null) {
return null;
}
if (item.getUserId() != null && !item.getUserId().isBlank()) {
return item.getUserId().trim();
}
if (item.getSpeakerId() == null || item.getSpeakerId().isBlank() || "-1".equals(item.getSpeakerId().trim())) {
return null;
}
return item.getSpeakerId().trim();
}
private String resolveSpeakerName(RealtimeMeetingTranscriptCacheItem item) {
if (item == null) {
return null;
}
if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) {
return item.getSpeakerName().trim();
}
String speakerId = resolveSpeakerId(item);
return speakerId == null || speakerId.isBlank() ? null : speakerId;
}
private String resolveSpeakerId(JsonNode sentence) {
String userId = readText(sentence, "user_id");
if (userId != null && !userId.isBlank()) {
return userId;
}
return readText(sentence, "speaker_id");
}
private Integer readInteger(JsonNode node, String fieldName) {
if (node == null || !node.has(fieldName) || !node.get(fieldName).canConvertToInt()) {
return null;
}
return node.get(fieldName).asInt();
}
private Integer readTimeMilliseconds(JsonNode node, String primaryField, String fallbackField) {
if (node == null) {
return null;
}
if (node.has(primaryField) && node.get(primaryField).canConvertToInt()) {
return node.get(primaryField).asInt();
}
if (node.has(fallbackField) && node.get(fallbackField).isNumber()) {
return Math.round((float) (node.get(fallbackField).asDouble() * 1000));
}
return null;
}
private String readText(JsonNode node, String fieldName) {
if (node == null || !node.has(fieldName) || node.get(fieldName).isNull()) {
return null;
}
String value = node.get(fieldName).asText("");
return value == null ? null : value.trim();
}
private long countNonEmptyItems(RealtimeMeetingTranscriptCacheState state) {
if (state.getItems() == null || state.getItems().isEmpty()) {
return 0L;
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.count();
}
private int nextSortOrder(RealtimeMeetingTranscriptCacheState state) {
Integer current = state.getNextSortOrder();
int next = current == null ? 0 : current;
state.setNextSortOrder(next + 1);
return next;
}
private int nextLegacySequence(RealtimeMeetingTranscriptCacheState state) {
Integer current = state.getNextLegacySequence();
int next = current == null ? 0 : current;
state.setNextLegacySequence(next + 1);
return next;
}
private RealtimeMeetingTranscriptCacheState getOrCreateState(Long meetingId) {
RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId);
if (state != null) {
if (state.getItems() == null) {
state.setItems(new ArrayList<>());
}
return state;
}
RealtimeMeetingTranscriptCacheState next = new RealtimeMeetingTranscriptCacheState();
next.setMeetingId(meetingId);
next.setItems(new ArrayList<>());
next.setNextSortOrder(0);
next.setNextLegacySequence(0);
next.setUpdatedAt(System.currentTimeMillis());
return next;
}
private Object lockForMeeting(Long meetingId) {
return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object());
}
}

View File

@ -0,0 +1,506 @@
package com.imeeting.service.realtime.impl;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.tencent.asrspeaker.SpeakerConstant;
import com.tencent.asrspeaker.SpeakerRecognitionListener;
import com.tencent.asrspeaker.SpeakerRecognitionResponse;
import com.tencent.asrspeaker.SpeakerRecognizer;
import com.tencent.asrspeaker.SpeakerRecognizerRequest;
import com.tencent.asrspeaker.SpeakerSentenceItem;
import com.tencent.core.ws.Credential;
import com.tencent.core.ws.SpeechClient;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.CloseStatus;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
@RequiredArgsConstructor
public class TencentRealtimeAsrChannel implements RealtimeAsrChannel {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String TARGET_WS_URL = "tencent-sdk://speaker-recognizer";
private static final String MEDIA_TENCENT_APP_ID = "tencentAppId";
private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId";
private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey";
private static final String STATE_CONNECTED = "tencentConnected";
private static final String STATE_STARTED = "tencentStarted";
private static final String STATE_RECOGNIZER = "tencentRecognizer";
private static final String STATE_SPEECH_CLIENT = "tencentSpeechClient";
private static final String STATE_STOP_REQUESTED = "tencentStopRequested";
private static final String STATE_MEETING_COMPLETE_REQUESTED = "tencentMeetingCompleteRequested";
private static final String STATE_FRONTEND_DETACHED = "tencentFrontendDetached";
private static final String STATE_SPEAKER_CONTEXT_ID = "speakerContextId";
private static final String STATE_VOICE_ID = "voiceId";
private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final String STATE_MODEL_CODE = "modelCode";
private static final String STATE_MEDIA_CONFIG = "mediaConfig";
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
@Override
public boolean supports(String provider) {
return ModelProviderEnum.TENCENT.getCode().equalsIgnoreCase(provider);
}
@Override
public String resolveTargetWsUrl(AiModelVO model) {
return TARGET_WS_URL;
}
@Override
public Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> payload = new HashMap<>();
payload.put("provider", ModelProviderEnum.TENCENT.getCode());
payload.put("engine_model_type", model.getModelCode());
payload.put("language", language);
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
root.put("payload", payload);
return root;
}
@Override
public void connect(RealtimeAsrChannelContext context) throws Exception {
String connectionId = currentConnectionId(context);
if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) {
context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接");
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接"));
return;
}
context.getChannelState().put(STATE_CONNECTED, Boolean.TRUE);
context.getChannelState().put(STATE_STARTED, Boolean.FALSE);
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.FALSE);
context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.FALSE);
context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.FALSE);
context.getChannelState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new java.util.ArrayList<byte[]>());
context.getCallback().onChannelOpen(context.getMeetingId());
}
@Override
public void handleFrontendText(RealtimeAsrChannelContext context, String payload) {
if (looksLikeStartMessage(payload)) {
startRecognizerIfNecessary(context);
return;
}
if (looksLikeStopMessage(payload)) {
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
}
@Override
public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (payload == null || payload.length == 0) {
return;
}
if (recognizer == null) {
queuePendingAudioFrame(context, payload);
return;
}
try {
recognizer.write(payload);
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 音频发送失败", ex);
}
}
@Override
public void closeMeeting(RealtimeAsrChannelContext context) {
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
@Override
public boolean isOpen(RealtimeAsrChannelContext context) {
return !Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED));
}
@Override
public void onFrontendDetached(RealtimeAsrChannelContext context) {
context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.TRUE);
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
static String buildFrontendTranscriptMessage(String sentenceKey,
String text,
boolean isFinal,
Integer sentenceId,
Long startTime,
Long endTime,
Integer speakerId) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", text);
data.put("is_final", isFinal);
if (sentenceId != null) {
data.put("sentence_id", sentenceId);
}
if (sentenceKey != null && !sentenceKey.isBlank()) {
data.put("sentence_key", sentenceKey);
}
if (startTime != null) {
data.put("start", startTime / 1000D);
}
if (endTime != null) {
data.put("end", endTime / 1000D);
}
if (speakerId != null) {
data.put("speaker_id", String.valueOf(speakerId));
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private void startRecognizerIfNecessary(RealtimeAsrChannelContext context) {
synchronized (context.getChannelState()) {
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_STARTED))) {
return;
}
try {
SpeechClient speechClient = createSpeechClient();
SpeakerRecognizerRequest request = createRecognizerRequest(context);
SpeakerRecognizer recognizer = createRecognizer(context, speechClient, request);
context.getChannelState().put(STATE_SPEECH_CLIENT, speechClient);
context.getChannelState().put(STATE_RECOGNIZER, recognizer);
context.getChannelState().put(STATE_VOICE_ID, request.getVoiceId());
recognizer.start();
context.getChannelState().put(STATE_STARTED, Boolean.TRUE);
flushPendingAudioFrames(context, recognizer);
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_CONNECT_FAILED", "腾讯实时 ASR 启动失败", ex);
}
}
}
protected SpeechClient createSpeechClient() {
return new SpeechClient(SpeakerConstant.DEFAULT_RT_REQ_URL);
}
protected SpeakerRecognizer createRecognizer(RealtimeAsrChannelContext context,
SpeechClient speechClient,
SpeakerRecognizerRequest request) {
return new SpeakerRecognizer(
speechClient,
buildCredential(context),
request,
new TencentRecognitionListener(context)
);
}
private Credential buildCredential(RealtimeAsrChannelContext context) {
Map<String, Object> mediaConfig = getMediaConfig(context);
String appId = readConfigString(mediaConfig, MEDIA_TENCENT_APP_ID);
String secretId = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_ID);
String secretKey = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_KEY);
if (appId == null || secretId == null || secretKey == null) {
throw new RuntimeException("腾讯实时 ASR 会话缺少鉴权配置");
}
return new Credential(appId, secretId, secretKey);
}
SpeakerRecognizerRequest createRecognizerRequest(RealtimeAsrChannelContext context) {
SpeakerRecognizerRequest request = SpeakerRecognizerRequest.init();
request.setEngineModelType(resolveEngineModelType(context));
request.setVoiceFormat(SpeakerConstant.AUDIO_FORMAT_PCM);
request.setVoiceId(UUID.randomUUID().toString());
//是否需要vad
request.setNeedVad(1);
//vad静默时间
request.setVadSilenceTime(1000);
// 分句策略参数 0小1大
request.setSentenceStrategy(0);
//是否进行阿拉伯数字智能转换 0否1智能 23:打开数学相关转化
request.setConvertNumMode(1);
request.setSpeakerDiarization(1);
//启动断点续传
request.setEnableSpeakerContext(1);
String speakerContextId = resolveSpeakerContextId(context);
if (speakerContextId != null) {
request.setSpeakerContextId(speakerContextId);
}
return request;
}
private String resolveEngineModelType(RealtimeAsrChannelContext context) {
Object modelCode = context.getChannelState().get(STATE_MODEL_CODE);
if (modelCode instanceof String value && !value.isBlank()) {
return value;
}
return "16k_zh";
}
private String resolveSpeakerContextId(RealtimeAsrChannelContext context) {
Object speakerContextId = context.getChannelState().get(STATE_SPEAKER_CONTEXT_ID);
if (speakerContextId instanceof String value && !value.isBlank()) {
return value;
}
var status = realtimeMeetingSessionStateService.getStatus(context.getMeetingId());
if (status == null || status.getResumeConfig() == null) {
return null;
}
String value = status.getResumeConfig().getSpeakerContextId();
if (value == null || value.isBlank()) {
return null;
}
context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, value);
return value;
}
@SuppressWarnings("unchecked")
private Map<String, Object> getMediaConfig(RealtimeAsrChannelContext context) {
Object mediaConfig = context.getChannelState().get(STATE_MEDIA_CONFIG);
if (mediaConfig instanceof Map<?, ?> map) {
return (Map<String, Object>) map;
}
return Map.of();
}
private String readConfigString(Map<String, Object> mediaConfig, String key) {
Object value = mediaConfig.get(key);
if (value == null) {
return null;
}
String text = String.valueOf(value).trim();
return text.isEmpty() ? null : text;
}
private SpeakerRecognizer getRecognizer(RealtimeAsrChannelContext context) {
Object recognizer = context.getChannelState().get(STATE_RECOGNIZER);
return recognizer instanceof SpeakerRecognizer value ? value : null;
}
private SpeechClient getSpeechClient(RealtimeAsrChannelContext context) {
Object speechClient = context.getChannelState().get(STATE_SPEECH_CLIENT);
return speechClient instanceof SpeechClient value ? value : null;
}
private void stopRecognizer(RealtimeAsrChannelContext context) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (recognizer == null) {
shutdownSdkResources(context);
return;
}
try {
recognizer.stop();
} catch (Exception ex) {
log.warn("Tencent realtime ASR stop failed, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context), ex);
shutdownSdkResources(context);
}
}
private void shutdownSdkResources(RealtimeAsrChannelContext context) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (recognizer != null) {
try {
recognizer.close();
} catch (Exception ignored) {
// ignore
}
}
SpeechClient speechClient = getSpeechClient(context);
if (speechClient != null) {
try {
speechClient.shutdown();
} catch (Exception ignored) {
// ignore
}
}
context.getChannelState().remove(STATE_RECOGNIZER);
context.getChannelState().remove(STATE_SPEECH_CLIENT);
context.getChannelState().put(STATE_STARTED, Boolean.FALSE);
}
private void forwardResponse(RealtimeAsrChannelContext context,
SpeakerRecognitionResponse response,
boolean forceFinal) {
if (response == null || response.getSentences() == null || response.getSentences().getSentenceList() == null) {
return;
}
try {
rememberSpeakerContext(context, response);
String cachePayload = buildCachePayload(response, forceFinal);
realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), cachePayload);
for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) {
if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) {
continue;
}
boolean isFinal = forceFinal || item.getSentenceType() == 1;
context.getCallback().sendFrontendText(
context.getMeetingId(),
buildFrontendTranscriptMessage(
buildSentenceKey(context, item.getSentenceId()),
item.getSentence().trim(),
isFinal,
item.getSentenceId(),
item.getStartTime(),
item.getEndTime(),
item.getSpeakerId()
)
);
}
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 结果转发失败", ex);
}
}
private void rememberSpeakerContext(RealtimeAsrChannelContext context, SpeakerRecognitionResponse response) {
if (response == null || response.getSpeakerContextId() == null || response.getSpeakerContextId().isBlank()) {
return;
}
String speakerContextId = response.getSpeakerContextId().trim();
context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, speakerContextId);
realtimeMeetingSessionStateService.rememberSpeakerContext(context.getMeetingId(), speakerContextId);
}
private String buildSentenceKey(RealtimeAsrChannelContext context, Integer sentenceId) {
Object voiceId = context.getChannelState().get(STATE_VOICE_ID);
if (!(voiceId instanceof String value) || value.isBlank() || sentenceId == null) {
return null;
}
return value + "-" + sentenceId;
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) {
Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES);
if (frames instanceof List<?> list) {
((List<byte[]>) list).add(payload.clone());
return;
}
List<byte[]> next = new java.util.ArrayList<>();
next.add(payload.clone());
context.getChannelState().put(STATE_PENDING_AUDIO_FRAMES, next);
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(RealtimeAsrChannelContext context, SpeakerRecognizer recognizer) {
Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES);
if (!(frames instanceof List<?> list) || list.isEmpty()) {
return;
}
List<byte[]> pendingFrames = (List<byte[]>) list;
for (byte[] frame : pendingFrames) {
if (frame != null && frame.length > 0) {
recognizer.write(frame);
}
}
pendingFrames.clear();
}
private String buildCachePayload(SpeakerRecognitionResponse response, boolean forceFinal) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", forceFinal ? "end" : "sentences");
ArrayNode sentences = root.putArray("sentences");
for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) {
if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) {
continue;
}
ObjectNode sentenceNode = sentences.addObject();
sentenceNode.put("sentence", item.getSentence().trim());
sentenceNode.put("sentence_type", forceFinal ? 1 : item.getSentenceType());
sentenceNode.put("sentence_id", item.getSentenceId());
sentenceNode.put("speaker_id", String.valueOf(item.getSpeakerId()));
sentenceNode.put("start_time", item.getStartTime());
sentenceNode.put("end_time", item.getEndTime());
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private void handleChannelFailure(RealtimeAsrChannelContext context, String code, String message, Exception ex) {
log.error("Tencent realtime ASR channel failed, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context), ex);
shutdownSdkResources(context);
context.getCallback().sendFrontendError(context.getMeetingId(), code, message);
CompletableFuture.delayedExecutor(200, TimeUnit.MILLISECONDS).execute(
() -> context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR)
);
}
private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId();
}
private static boolean looksLikeStartMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private static boolean looksLikeStopMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"stop\"");
}
private final class TencentRecognitionListener extends SpeakerRecognitionListener {
private final RealtimeAsrChannelContext context;
private TencentRecognitionListener(RealtimeAsrChannelContext context) {
this.context = context;
}
@Override
public void onRecognitionStart(SpeakerRecognitionResponse response) {
rememberSpeakerContext(context, response);
log.info("Tencent realtime ASR started, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context));
}
@Override
public void onRecognitionSentences(SpeakerRecognitionResponse response) {
forwardResponse(context, response, false);
}
@Override
public void onSentenceEnd(SpeakerRecognitionResponse response) {
forwardResponse(context, response, true);
shutdownSdkResources(context);
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED))) {
context.getCallback().removeMeetingSession(context.getMeetingId());
}
}
@Override
public void onFail(SpeakerRecognitionResponse response, Exception error) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 识别失败", error);
}
}
}

View File

@ -0,0 +1,39 @@
package com.imeeting.support.redis;
import com.imeeting.common.RedisKeys;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.support.RedisSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.time.Duration;
@Component
@RequiredArgsConstructor
public class RealtimeMeetingTranscriptCache {
private static final Duration CACHE_TTL = Duration.ofHours(12);
private final RedisSupport redisSupport;
public RealtimeMeetingTranscriptCacheState getState(Long meetingId) {
if (meetingId == null) {
return null;
}
return redisSupport.getJsonQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId), RealtimeMeetingTranscriptCacheState.class);
}
public void saveState(RealtimeMeetingTranscriptCacheState state) {
if (state == null || state.getMeetingId() == null) {
return;
}
redisSupport.setJson(RedisKeys.realtimeMeetingTranscriptCacheKey(state.getMeetingId()), state, CACHE_TTL);
}
public void clear(Long meetingId) {
if (meetingId == null) {
return;
}
redisSupport.deleteQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId));
}
}

View File

@ -1,173 +1,133 @@
package com.imeeting.websocket; package com.imeeting.websocket;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.dto.biz.RealtimeSocketSessionData; import com.imeeting.dto.biz.RealtimeSocketSessionData;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.biz.RealtimeMeetingSocketSessionService; import com.imeeting.service.biz.RealtimeMeetingSocketSessionService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelCallback;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeAsrChannelFactory;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.imeeting.service.realtime.impl.LocalRealtimeAsrChannel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.socket.BinaryMessage; import org.springframework.web.socket.BinaryMessage;
import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.PingMessage;
import org.springframework.web.socket.PongMessage; import org.springframework.web.socket.PongMessage;
import org.springframework.web.socket.TextMessage; import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.AbstractWebSocketHandler; import org.springframework.web.socket.handler.AbstractWebSocketHandler;
import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator; import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator;
import java.io.ByteArrayOutputStream;
import java.net.URI; import java.net.URI;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@Slf4j @Slf4j
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandler { public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandler {
private static final String ATTR_FRONTEND_SESSION = "frontendSession";
private static final String ATTR_UPSTREAM_SOCKET = "upstreamSocket";
private static final String ATTR_MEETING_ID = "meetingId"; private static final String ATTR_MEETING_ID = "meetingId";
private static final String ATTR_TARGET_WS_URL = "targetWsUrl"; private static final String ATTR_TARGET_WS_URL = "targetWsUrl";
private static final String ATTR_PROVIDER = "provider";
private static final String ATTR_FRONTEND_TEXT_COUNT = "frontendTextCount"; private static final String ATTR_FRONTEND_TEXT_COUNT = "frontendTextCount";
private static final String ATTR_FRONTEND_BINARY_COUNT = "frontendBinaryCount"; private static final String ATTR_FRONTEND_BINARY_COUNT = "frontendBinaryCount";
private static final String ATTR_UPSTREAM_SEND_CHAIN = "upstreamSendChain"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String ATTR_START_MESSAGE_SENT = "startMessageSent";
private static final String ATTR_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final CompletableFuture<Void> COMPLETED = CompletableFuture.completedFuture(null);
private final RealtimeMeetingSocketSessionService realtimeMeetingSocketSessionService; private final RealtimeMeetingSocketSessionService realtimeMeetingSocketSessionService;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService; private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
private final RealtimeAsrChannelFactory realtimeAsrChannelFactory;
private final ConcurrentMap<Long, MeetingChannelSession> meetingSessions = new ConcurrentHashMap<>();
private final ConcurrentMap<Long, Object> meetingLocks = new ConcurrentHashMap<>();
@Override @Override
public void afterConnectionEstablished(WebSocketSession session) throws Exception { public void afterConnectionEstablished(WebSocketSession session) throws Exception {
String sessionToken = extractQueryParam(session.getUri(), "sessionToken"); String sessionToken = extractQueryParam(session.getUri(), "sessionToken");
RealtimeSocketSessionData sessionData = realtimeMeetingSocketSessionService.getSessionData(sessionToken); RealtimeSocketSessionData sessionData = realtimeMeetingSocketSessionService.getSessionData(sessionToken);
if (sessionData == null) { if (sessionData == null) {
log.warn("Realtime websocket rejected: invalid session token, sessionId={}", session.getId()); log.warn("实时会议 websocket 拒绝连接:会话令牌无效,sessionId={}", session.getId());
session.close(CloseStatus.POLICY_VIOLATION.withReason("实时 Socket 会话无效")); session.close(CloseStatus.POLICY_VIOLATION.withReason("实时 Socket 会话无效"));
return; return;
} }
ConcurrentWebSocketSessionDecorator frontendSession = ConcurrentWebSocketSessionDecorator frontendSession =
new ConcurrentWebSocketSessionDecorator(session, (int) Duration.ofSeconds(15).toMillis(), 1024 * 1024); new ConcurrentWebSocketSessionDecorator(session, (int) Duration.ofSeconds(15).toMillis(), 1024 * 1024);
session.getAttributes().put(ATTR_FRONTEND_SESSION, frontendSession);
session.getAttributes().put(ATTR_MEETING_ID, sessionData.getMeetingId()); session.getAttributes().put(ATTR_MEETING_ID, sessionData.getMeetingId());
session.getAttributes().put(ATTR_TARGET_WS_URL, sessionData.getTargetWsUrl()); session.getAttributes().put(ATTR_TARGET_WS_URL, sessionData.getTargetWsUrl());
session.getAttributes().put(ATTR_PROVIDER, sessionData.getProvider());
session.getAttributes().put(ATTR_FRONTEND_TEXT_COUNT, new AtomicInteger()); session.getAttributes().put(ATTR_FRONTEND_TEXT_COUNT, new AtomicInteger());
session.getAttributes().put(ATTR_FRONTEND_BINARY_COUNT, new AtomicInteger()); session.getAttributes().put(ATTR_FRONTEND_BINARY_COUNT, new AtomicInteger());
session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED);
session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.FALSE);
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
realtimeMeetingAudioStorageService.openSession(sessionData.getMeetingId(), session.getId()); realtimeMeetingAudioStorageService.openSession(sessionData.getMeetingId(), session.getId());
log.info("Realtime websocket accepted: meetingId={}, sessionId={}, upstream={}", log.info("实时会议 websocket 已接入meetingId={}, sessionId={}, provider={}, upstream={}",
sessionData.getMeetingId(), session.getId(), sessionData.getTargetWsUrl()); sessionData.getMeetingId(), session.getId(), sessionData.getProvider(), sessionData.getTargetWsUrl());
java.net.http.WebSocket upstreamSocket; attachFrontendSession(sessionData, session, frontendSession);
try {
upstreamSocket = java.net.http.HttpClient.newHttpClient()
.newWebSocketBuilder()
.buildAsync(URI.create(sessionData.getTargetWsUrl()),
new UpstreamListener(
frontendSession,
session,
sessionData.getMeetingId(),
sessionData.getTargetWsUrl(),
realtimeMeetingSessionStateService
))
.get();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.error("Realtime websocket upstream connect interrupted: meetingId={}, sessionId={}",
sessionData.getMeetingId(), session.getId(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接第三方识别服务时被中断");
realtimeMeetingAudioStorageService.closeSession(session.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断"));
return;
} catch (ExecutionException | CompletionException ex) {
log.warn("Failed to connect upstream websocket, meetingId={}, target={}", sessionData.getMeetingId(), sessionData.getTargetWsUrl(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接第三方识别服务失败,请检查模型 WebSocket 配置或服务状态");
realtimeMeetingAudioStorageService.closeSession(session.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败"));
return;
}
session.getAttributes().put(ATTR_UPSTREAM_SOCKET, upstreamSocket);
} }
@Override @Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) { protected void handleTextMessage(WebSocketSession session, TextMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); MeetingChannelSession meetingSession = getMeetingSession(session);
if (upstreamSocket == null) { if (meetingSession == null || !meetingSession.isChannelOpen()) {
log.warn("Frontend text ignored because upstream socket is unavailable, meetingId={}, sessionId={}", log.warn("前端文本消息已忽略:上游 ASR 连接不可用meetingId={}, sessionId={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId()); session.getAttributes().get(ATTR_MEETING_ID), session.getId());
return; return;
} }
int count = nextCount(session, ATTR_FRONTEND_TEXT_COUNT); int count = nextCount(session, ATTR_FRONTEND_TEXT_COUNT);
log.info("Frontend text -> upstream: meetingId={}, sessionId={}, count={}, payload={}", String payload = message.getPayload();
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, summarizeText(message.getPayload())); log.info("前端文本 -> ASR 渠道meetingId={}, sessionId={}, provider={}, count={}, payload={}",
sendUpstreamOrdered(session, () -> upstreamSocket.sendText(message.getPayload(), true), "text"); session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, summarizeText(payload));
if (looksLikeStartMessage(message.getPayload())) { meetingSession.channel.handleFrontendText(meetingSession.context, payload);
session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.TRUE);
flushPendingAudioFrames(session, upstreamSocket);
}
} }
@Override @Override
protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); MeetingChannelSession meetingSession = getMeetingSession(session);
if (upstreamSocket == null) { if (meetingSession == null || !meetingSession.isChannelOpen()) {
log.warn("Frontend binary ignored because upstream socket is unavailable, meetingId={}, sessionId={}", log.warn("前端音频帧已忽略:上游 ASR 连接不可用meetingId={}, sessionId={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId()); session.getAttributes().get(ATTR_MEETING_ID), session.getId());
return; return;
} }
int count = nextCount(session, ATTR_FRONTEND_BINARY_COUNT); int count = nextCount(session, ATTR_FRONTEND_BINARY_COUNT);
int bytes = message.getPayloadLength(); int bytes = message.getPayloadLength();
if (shouldLogBinaryFrame(count)) { if (shouldLogBinaryFrame(count)) {
log.info("Frontend binary -> upstream: meetingId={}, sessionId={}, count={}, bytes={}", log.info("前端音频帧 -> ASR 渠道meetingId={}, sessionId={}, provider={}, count={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes); session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, bytes);
} }
byte[] payload = toByteArray(message.getPayload()); byte[] payload = toByteArray(message.getPayload());
realtimeMeetingAudioStorageService.append(session.getId(), payload); realtimeMeetingAudioStorageService.append(session.getId(), payload);
if (!Boolean.TRUE.equals(session.getAttributes().get(ATTR_START_MESSAGE_SENT))) { meetingSession.channel.handleFrontendBinary(meetingSession.context, payload);
queuePendingAudioFrame(session, payload);
if (shouldLogBinaryFrame(count)) {
log.warn("Frontend binary queued before start message: meetingId={}, sessionId={}, count={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes);
}
return;
}
sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary");
} }
@Override @Override
protected void handlePongMessage(WebSocketSession session, PongMessage message) { protected void handlePongMessage(WebSocketSession session, PongMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); if (getMeetingSession(session) == null) {
if (upstreamSocket == null) {
return; return;
} }
sendUpstreamOrdered(session, () -> upstreamSocket.sendPong(copyBuffer(message.getPayload())), "pong"); log.debug("前端 pong 已在本地忽略meetingId={}, sessionId={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), message.getPayloadLength());
} }
@Override @Override
public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception { public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception {
log.error("Realtime websocket transport error: meetingId={}, sessionId={}, upstream={}", log.error("实时会议 websocket 传输异常:meetingId={}, sessionId={}, upstream={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_TARGET_WS_URL), exception); session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_TARGET_WS_URL), exception);
closeUpstreamSocket(session, CloseStatus.SERVER_ERROR); detachFrontend(session);
realtimeMeetingAudioStorageService.closeSession(session.getId());
if (session.isOpen()) { if (session.isOpen()) {
session.close(CloseStatus.SERVER_ERROR); session.close(CloseStatus.SERVER_ERROR);
} }
@ -175,32 +135,160 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
@Override @Override
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
log.info("Realtime websocket closed: meetingId={}, sessionId={}, code={}, reason={}", log.info("实时会议 websocket 已关闭:meetingId={}, sessionId={}, code={}, reason={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), status.getCode(), status.getReason()); session.getAttributes().get(ATTR_MEETING_ID), session.getId(), status.getCode(), status.getReason());
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID); Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (meetingIdValue instanceof Long meetingId) { if (meetingIdValue instanceof Long meetingId) {
detachFrontend(meetingId, session.getId());
realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, session.getId()); realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, session.getId());
} }
realtimeMeetingAudioStorageService.closeSession(session.getId()); realtimeMeetingAudioStorageService.closeSession(session.getId());
closeUpstreamSocket(session, status);
} }
private java.net.http.WebSocket getUpstreamSocket(WebSocketSession session) { public void closeMeetingSession(Long meetingId) {
Object socket = session.getAttributes().get(ATTR_UPSTREAM_SOCKET); if (meetingId == null) {
if (socket instanceof java.net.http.WebSocket webSocket) { return;
return webSocket; }
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null || meetingSession.channel == null) {
return;
}
meetingSession.channel.closeMeeting(meetingSession.context);
}
private void attachFrontendSession(RealtimeSocketSessionData sessionData,
WebSocketSession rawSession,
ConcurrentWebSocketSessionDecorator frontendSession) throws Exception {
Long meetingId = sessionData.getMeetingId();
MeetingChannelSession meetingSession;
boolean reused = false;
synchronized (lockForMeeting(meetingId)) {
meetingSession = meetingSessions.get(meetingId);
if (meetingSession != null && meetingSession.isChannelOpen()) {
String previousSessionId = meetingSession.context.getRawSession() == null
? null
: meetingSession.context.getRawSession().getId();
meetingSession.clearFrontendIfClosed();
if (previousSessionId != null && !meetingSession.hasOpenFrontend()) {
realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, previousSessionId);
} }
return null; if (meetingSession.hasOpenFrontend()) {
sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃前端连接");
frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("已存在活跃的前端连接"));
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
return;
}
if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) {
sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_REJECTED", "当前状态下无法继续会议");
frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("当前状态下无法继续会议"));
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
return;
}
meetingSession.bindFrontend(rawSession, frontendSession);
reused = true;
} else {
RealtimeAsrChannel channel = realtimeAsrChannelFactory.getRequired(sessionData.getProvider());
RealtimeAsrChannelContext context = new RealtimeAsrChannelContext();
context.setMeetingId(meetingId);
context.setProvider(realtimeAsrChannelFactory.normalizeProvider(sessionData.getProvider()));
context.setTargetWsUrl(sessionData.getTargetWsUrl());
context.setCallback(new HandlerChannelCallback());
context.bindFrontendSession(rawSession, frontendSession);
context.getChannelState().put("modelCode", sessionData.getModelCode());
context.getChannelState().put("mediaConfig", sessionData.getMediaConfig());
meetingSession = new MeetingChannelSession(meetingId, channel, context);
meetingSessions.put(meetingId, meetingSession);
}
} }
private void closeUpstreamSocket(WebSocketSession session, CloseStatus status) { if (reused) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); sendProxyReady(frontendSession);
if (upstreamSocket != null) { replayCachedMessages(meetingId, frontendSession);
upstreamSocket.sendClose(status.getCode(), status.getReason() == null ? "" : status.getReason()); return;
session.getAttributes().remove(ATTR_UPSTREAM_SOCKET); }
try {
meetingSession.channel.connect(meetingSession.context);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
removeMeetingSession(meetingId, meetingSession);
log.error("连接上游 ASR websocket 时被中断meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接上游 ASR 服务时被中断");
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断"));
} catch (Exception ex) {
removeMeetingSession(meetingId, meetingSession);
log.warn("连接上游 ASR websocket 失败meetingId={}, provider={}, target={}",
meetingId, sessionData.getProvider(), sessionData.getTargetWsUrl(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接上游 ASR 服务失败");
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败"));
}
}
private void replayCachedMessages(Long meetingId, ConcurrentWebSocketSessionDecorator frontendSession) {
try {
if (!frontendSession.isOpen()) {
return;
}
for (RealtimeMeetingTranscriptCacheItem item : realtimeMeetingTranscriptCacheService.listOrderedItems(meetingId)) {
frontendSession.sendMessage(new TextMessage(LocalRealtimeAsrChannel.buildFrontendTranscriptMessage(item)));
}
} catch (Exception ex) {
log.warn("回放缓存转写消息失败meetingId={}", meetingId, ex);
}
}
private void sendProxyReady(ConcurrentWebSocketSessionDecorator frontendSession) throws Exception {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}"));
}
}
private void detachFrontend(WebSocketSession session) {
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (meetingIdValue instanceof Long meetingId) {
detachFrontend(meetingId, session.getId());
}
}
private void detachFrontend(Long meetingId, String sessionId) {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
synchronized (lockForMeeting(meetingId)) {
if (meetingSession.context.getRawSession() != null && meetingSession.context.getRawSession().getId().equals(sessionId)) {
meetingSession.channel.onFrontendDetached(meetingSession.context);
}
meetingSession.detachFrontend(sessionId);
}
}
private MeetingChannelSession getMeetingSession(WebSocketSession session) {
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (!(meetingIdValue instanceof Long meetingId)) {
return null;
}
return meetingSessions.get(meetingId);
}
void removeMeetingSession(Long meetingId) {
synchronized (lockForMeeting(meetingId)) {
meetingSessions.remove(meetingId);
}
}
void removeMeetingSession(Long meetingId, MeetingChannelSession meetingSession) {
synchronized (lockForMeeting(meetingId)) {
meetingSessions.remove(meetingId, meetingSession);
} }
} }
private Object lockForMeeting(Long meetingId) {
return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object());
}
private String extractQueryParam(URI uri, String key) { private String extractQueryParam(URI uri, String key) {
if (uri == null || uri.getQuery() == null || uri.getQuery().isBlank()) { if (uri == null || uri.getQuery() == null || uri.getQuery().isBlank()) {
return null; return null;
@ -213,13 +301,6 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
.orElse(null); .orElse(null);
} }
private ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
private byte[] toByteArray(ByteBuffer source) { private byte[] toByteArray(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer(); ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()]; byte[] bytes = new byte[duplicate.remaining()];
@ -235,21 +316,18 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
return 0; return 0;
} }
@SuppressWarnings("unchecked") private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) {
private void sendUpstreamOrdered(WebSocketSession session, Supplier<CompletableFuture<?>> sendAction, String messageType) { try {
synchronized (session) { if (!frontendSession.isOpen()) {
CompletableFuture<Void> chain = (CompletableFuture<Void>) session.getAttributes() return;
.getOrDefault(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED); }
CompletableFuture<Void> nextChain = chain Map<String, Object> payload = new HashMap<>();
.exceptionally(ex -> null) payload.put("type", "error");
.thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null)); payload.put("code", code);
nextChain = nextChain.whenComplete((ignored, ex) -> { payload.put("message", message);
if (ex != null) { frontendSession.sendMessage(new TextMessage(OBJECT_MAPPER.writeValueAsString(payload)));
log.error("Ordered upstream send failed: meetingId={}, sessionId={}, type={}", } catch (Exception ex) {
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), messageType, ex); log.warn("向前端发送实时代理错误消息失败code={}", code, ex);
}
});
session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, nextChain);
} }
} }
@ -268,237 +346,113 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
return normalized.substring(0, 240) + "..."; return normalized.substring(0, 240) + "...";
} }
private boolean looksLikeStartMessage(String payload) { static final class MeetingChannelSession {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) {
try {
if (!frontendSession.isOpen()) {
return;
}
Map<String, Object> payload = new HashMap<>();
payload.put("type", "error");
payload.put("code", code);
payload.put("message", message);
frontendSession.sendMessage(new TextMessage(new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(payload)));
} catch (Exception ex) {
log.warn("Failed to send realtime proxy error to frontend: code={}", code, ex);
}
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(WebSocketSession session, byte[] payload) {
synchronized (session) {
List<byte[]> pendingFrames = (List<byte[]>) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES);
if (pendingFrames == null) {
pendingFrames = new ArrayList<>();
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, pendingFrames);
}
pendingFrames.add(payload);
}
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(WebSocketSession session, java.net.http.WebSocket upstreamSocket) {
List<byte[]> pendingFrames;
synchronized (session) {
pendingFrames = (List<byte[]>) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES);
if (pendingFrames == null || pendingFrames.isEmpty()) {
return;
}
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
log.info("Flushing queued audio frames after start message: meetingId={}, sessionId={}, frameCount={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), pendingFrames.size());
for (byte[] frame : pendingFrames) {
sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush");
}
}
private static final class UpstreamListener implements java.net.http.WebSocket.Listener {
private final ConcurrentWebSocketSessionDecorator frontendSession;
private final WebSocketSession rawSession;
private final Long meetingId; private final Long meetingId;
private final String targetWsUrl; private final RealtimeAsrChannel channel;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; private final RealtimeAsrChannelContext context;
private final StringBuilder textBuffer = new StringBuilder();
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
private final AtomicInteger upstreamTextCount = new AtomicInteger();
private final AtomicInteger upstreamBinaryCount = new AtomicInteger();
private UpstreamListener(ConcurrentWebSocketSessionDecorator frontendSession, WebSocketSession rawSession, private MeetingChannelSession(Long meetingId, RealtimeAsrChannel channel, RealtimeAsrChannelContext context) {
Long meetingId, String targetWsUrl,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService) {
this.frontendSession = frontendSession;
this.rawSession = rawSession;
this.meetingId = meetingId; this.meetingId = meetingId;
this.targetWsUrl = targetWsUrl; this.channel = channel;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService; this.context = context;
}
private void bindFrontend(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) {
context.bindFrontendSession(rawSession, frontendSession);
} }
private void detachFrontend(String sessionId) {
if (context.getRawSession() != null && context.getRawSession().getId().equals(sessionId)) {
context.bindFrontendSession(null, null);
}
}
private void clearFrontendIfClosed() {
if (context.getRawSession() != null && !context.getRawSession().isOpen()) {
context.bindFrontendSession(null, null);
}
}
private boolean hasOpenFrontend() {
return context.getFrontendSession() != null
&& context.getFrontendSession().isOpen()
&& context.getRawSession() != null
&& context.getRawSession().isOpen();
}
private boolean isChannelOpen() {
return channel != null && channel.isOpen(context);
}
}
private final class HandlerChannelCallback implements RealtimeAsrChannelCallback {
@Override @Override
public void onOpen(java.net.http.WebSocket webSocket) { public void onChannelOpen(Long meetingId) throws Exception {
log.info("Upstream websocket opened: meetingId={}, sessionId={}, upstream={}", MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
meetingId, rawSession.getId(), targetWsUrl); if (meetingSession == null) {
if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) {
sendFrontendError("REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃实时连接,请先关闭旧连接后再继续");
webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "Active realtime connection already exists");
closeFrontend(CloseStatus.POLICY_VIOLATION.withReason("已存在活动的实时连接"));
return; return;
} }
try { ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession.isOpen()) { if (frontendSession != null && frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}")); sendProxyReady(frontendSession);
} }
} catch (Exception ex) { }
log.error("Failed to notify frontend that upstream websocket is ready: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR); @Override
public void sendFrontendText(Long meetingId, String payload) throws Exception {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return; return;
} }
webSocket.request(1); ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession != null && frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage(payload));
}
} }
@Override @Override
public java.util.concurrent.CompletionStage<?> onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) { public void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception {
textBuffer.append(data); MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (last) { if (meetingSession == null) {
int count = upstreamTextCount.incrementAndGet(); return;
try { }
if (frontendSession.isOpen()) { ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
frontendSession.sendMessage(new TextMessage(textBuffer.toString())); if (frontendSession != null && frontendSession.isOpen()) {
} frontendSession.sendMessage(new BinaryMessage(payload));
log.info("Upstream text -> frontend: meetingId={}, sessionId={}, count={}, payload={}", }
meetingId, rawSession.getId(), count, summarizeText(textBuffer.toString()));
} catch (Exception ex) {
log.error("Failed to forward upstream text: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
} finally {
textBuffer.setLength(0);
}
}
webSocket.request(1);
return COMPLETED;
} }
@Override @Override
public java.util.concurrent.CompletionStage<?> onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) { public void sendFrontendError(Long meetingId, String code, String message) {
byte[] chunk = new byte[data.remaining()]; MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
data.get(chunk); if (meetingSession == null) {
binaryBuffer.writeBytes(chunk); return;
if (last) { }
int count = upstreamBinaryCount.incrementAndGet(); ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
try { if (frontendSession != null) {
if (frontendSession.isOpen()) { RealtimeMeetingProxyWebSocketHandler.this.sendFrontendError(frontendSession, code, message);
frontendSession.sendMessage(new BinaryMessage(binaryBuffer.toByteArray())); }
}
if (shouldLogBinaryFrame(count)) {
log.info("Upstream binary -> frontend: meetingId={}, sessionId={}, count={}, bytes={}",
meetingId, rawSession.getId(), count, binaryBuffer.size());
}
} catch (Exception ex) {
log.error("Failed to forward upstream binary: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
} finally {
binaryBuffer.reset();
}
}
webSocket.request(1);
return COMPLETED;
} }
@Override @Override
public java.util.concurrent.CompletionStage<?> onPing(java.net.http.WebSocket webSocket, ByteBuffer message) { public void removeMeetingSession(Long meetingId) {
RealtimeMeetingProxyWebSocketHandler.this.removeMeetingSession(meetingId);
}
@Override
public void closeFrontend(Long meetingId, CloseStatus status) {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
try { try {
if (frontendSession.isOpen()) { WebSocketSession rawSession = meetingSession.context.getRawSession();
frontendSession.sendMessage(new PingMessage(copyBuffer(message))); if (rawSession != null && rawSession.isOpen()) {
}
log.info("Upstream ping -> frontend: meetingId={}, sessionId={}, bytes={}",
meetingId, rawSession.getId(), message.remaining());
} catch (Exception ex) {
log.error("Failed to forward upstream ping: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPong(java.net.http.WebSocket webSocket, ByteBuffer message) {
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new PongMessage(copyBuffer(message)));
}
log.info("Upstream pong -> frontend: meetingId={}, sessionId={}, bytes={}",
meetingId, rawSession.getId(), message.remaining());
} catch (Exception ex) {
log.error("Failed to forward upstream pong: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) {
log.info("Upstream websocket closed: meetingId={}, sessionId={}, code={}, reason={}",
meetingId, rawSession.getId(), statusCode, reason);
sendFrontendError("REALTIME_UPSTREAM_CLOSED", reason == null || reason.isBlank() ? "第三方识别服务已断开连接" : "第三方识别服务已断开: " + reason);
closeFrontend(new CloseStatus(statusCode, reason));
return COMPLETED;
}
@Override
public void onError(java.net.http.WebSocket webSocket, Throwable error) {
log.error("Upstream websocket error: meetingId={}, sessionId={}, upstream={}",
meetingId, rawSession.getId(), targetWsUrl, error);
sendFrontendError("REALTIME_UPSTREAM_ERROR", error == null || error.getMessage() == null || error.getMessage().isBlank()
? "第三方识别服务连接异常"
: "第三方识别服务连接异常: " + error.getMessage());
closeFrontend(CloseStatus.SERVER_ERROR);
}
private void sendFrontendError(String code, String message) {
try {
if (!frontendSession.isOpen()) {
return;
}
frontendSession.sendMessage(new TextMessage("{\"type\":\"error\",\"code\":\"" + code + "\",\"message\":\"" + escapeJson(message) + "\"}"));
} catch (Exception ex) {
log.warn("Failed to send upstream error to frontend: meetingId={}, sessionId={}, code={}", meetingId, rawSession.getId(), code, ex);
}
}
private String escapeJson(String value) {
if (value == null) {
return "";
}
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\r", "\\r")
.replace("\n", "\\n");
}
private void closeFrontend(CloseStatus status) {
try {
if (rawSession.isOpen()) {
rawSession.close(status); rawSession.close(status);
} }
} catch (Exception ignored) { } catch (Exception ignored) {
// ignore close failure // ignore close failure
} }
} }
private ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
} }
} }

View File

@ -11,8 +11,23 @@ option java_outer_classname = "PushProto";
// ========================= // =========================
enum Platform { enum Platform {
PLATFORM_UNKNOWN = 0; PLATFORM_UNKNOWN = 0;
// Mobile
ANDROID = 1; ANDROID = 1;
IOS = 2; IOS = 2;
HARMONY_MOBILE = 3;
// Desktop
WINDOWS = 10;
MACOS = 11;
LINUX = 12;
// Linux
KYLIN = 20;
UOS = 21;
// Harmony PC
HARMONY_PC = 30;
} }
// ========================= // =========================

View File

@ -349,6 +349,64 @@ class AiModelServiceImplTest {
assertNull(captor.getValue().getApiKey()); assertNull(captor.getValue().getApiKey());
} }
@Test
void saveModelShouldRejectTencentAsrWithoutSecretKey() {
AiModelServiceImpl service = new AiModelServiceImpl(
objectMapper,
mock(AsrModelMapper.class),
mock(LlmModelMapper.class)
);
AiModelDTO dto = new AiModelDTO();
dto.setModelType("ASR");
dto.setModelName("tencent-asr");
dto.setProvider("tencent");
dto.setModelCode("16k_zh");
dto.setIsDefault(0);
dto.setStatus(1);
dto.setMediaConfig(Map.of(
"tencentAppId", "app-id",
"tencentSecretId", "secret-id"
));
RuntimeException ex = assertThrows(RuntimeException.class, () -> service.saveModel(dto));
assertEquals("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey", ex.getMessage());
}
@Test
void saveModelShouldPersistTencentAsrWithoutBaseUrl() {
AsrModelMapper asrModelMapper = mock(AsrModelMapper.class);
when(asrModelMapper.insert(any(AsrModel.class))).thenReturn(1);
AiModelServiceImpl service = new AiModelServiceImpl(
objectMapper,
asrModelMapper,
mock(LlmModelMapper.class)
);
AiModelDTO dto = new AiModelDTO();
dto.setModelType("ASR");
dto.setModelName("tencent-asr");
dto.setProvider("tencent");
dto.setModelCode("16k_zh");
dto.setIsDefault(0);
dto.setStatus(1);
dto.setMediaConfig(Map.of(
"tencentAppId", "app-id",
"tencentSecretId", "secret-id",
"tencentSecretKey", "secret-key"
));
service.saveModel(dto);
ArgumentCaptor<AsrModel> captor = ArgumentCaptor.forClass(AsrModel.class);
verify(asrModelMapper, times(1)).insert(captor.capture());
assertEquals("tencent", captor.getValue().getProvider());
assertEquals("16k_zh", captor.getValue().getModelCode());
assertEquals("secret-key", captor.getValue().getMediaConfig().get("tencentSecretKey"));
assertNull(captor.getValue().getBaseUrl());
}
private void captureRequest(HttpExchange exchange, private void captureRequest(HttpExchange exchange,
AtomicReference<String> requestPath, AtomicReference<String> requestPath,
AtomicReference<String> authorization, AtomicReference<String> authorization,

View File

@ -9,6 +9,7 @@ export type SummaryDetailLevel = "DETAILED" | "STANDARD" | "BRIEF";
export interface MeetingCreateConfig { export interface MeetingCreateConfig {
offlineEnabled: boolean; offlineEnabled: boolean;
realtimeEnabled: boolean; realtimeEnabled: boolean;
aiCatalogEnabled?: boolean;
offlineAudioMaxSizeMb: number; offlineAudioMaxSizeMb: number;
chunkUploadEnabled?: boolean; chunkUploadEnabled?: boolean;
chunkDurationSeconds?: number; chunkDurationSeconds?: number;
@ -35,6 +36,7 @@ export interface MeetingVO {
summaryDetailLevel?: SummaryDetailLevel; summaryDetailLevel?: SummaryDetailLevel;
summaryModelId: number; summaryModelId: number;
promptId?: number; promptId?: number;
aiCatalogEnabled?: boolean;
audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED"; audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED";
audioSaveMessage?: string; audioSaveMessage?: string;
accessPassword?: string; accessPassword?: string;
@ -247,12 +249,6 @@ export const createRealtimeMeeting = (data: CreateRealtimeMeetingCommand) => {
); );
}; };
export const appendRealtimeTranscripts = (meetingId: number, data: RealtimeTranscriptItemDTO[]) => {
return http.post<{ code: string; data: boolean; msg: string }>(
`/api/biz/meeting/${meetingId}/realtime/transcripts`,
data
);
};
export const getRealtimeMeetingSessionStatus = (meetingId: number) => { export const getRealtimeMeetingSessionStatus = (meetingId: number) => {
return http.get<{ code: string; data: RealtimeMeetingSessionStatus; msg: string }>( return http.get<{ code: string; data: RealtimeMeetingSessionStatus; msg: string }>(

View File

@ -138,6 +138,7 @@ export default function Users() {
const [filterTenantId, setFilterTenantId] = useState<number | undefined>(undefined); const [filterTenantId, setFilterTenantId] = useState<number | undefined>(undefined);
const [drawerOpen, setDrawerOpen] = useState(false); const [drawerOpen, setDrawerOpen] = useState(false);
const [editing, setEditing] = useState<SysUser | null>(null); const [editing, setEditing] = useState<SysUser | null>(null);
const [roleSelectOpen, setRoleSelectOpen] = useState(false);
const [form] = Form.useForm(); const [form] = Form.useForm();
const [resetPasswordOpen, setResetPasswordOpen] = useState(false); const [resetPasswordOpen, setResetPasswordOpen] = useState(false);
const [resetPasswordLoading, setResetPasswordLoading] = useState(false); const [resetPasswordLoading, setResetPasswordLoading] = useState(false);
@ -264,6 +265,7 @@ export default function Users() {
const openCreate = () => { const openCreate = () => {
setEditing(null); setEditing(null);
setRoleSelectOpen(false);
form.resetFields(); form.resetFields();
form.setFieldsValue({ form.setFieldsValue({
status: 1, status: 1,
@ -277,6 +279,7 @@ export default function Users() {
const openEdit = async (record: SysUser) => { const openEdit = async (record: SysUser) => {
setEditing(record); setEditing(record);
setRoleSelectOpen(false);
try { try {
const detail = await getUserDetail(record.userId); const detail = await getUserDetail(record.userId);
const roleIds = await listUserRoles(record.userId); const roleIds = await listUserRoles(record.userId);
@ -594,10 +597,13 @@ export default function Users() {
</Form.Item> </Form.Item>
) : null} ) : null}
<Form.Item label={t("users.roles")} name="roleIds" <Form.Item label={t("users.roles")} name="roleIds"
rules={[{required: true, message: t("users.roles")}]}><Select mode="multiple" rules={[{required: true, message: t("users.roles")}]}><Select mode="multiple"
placeholder={t("users.roles")} placeholder={t("users.roles")}
options={roleOptions} options={roleOptions}
optionFilterProp={isPlatformMode ? "searchText" : "label"}/></Form.Item> open={roleSelectOpen}
onDropdownVisibleChange={setRoleSelectOpen}
onChange={() => setRoleSelectOpen(false)}
optionFilterProp={isPlatformMode ? "searchText" : "label"}/></Form.Item>
{!isPlatformMode && <Form.Item label={t("users.orgNode")} name="orgId"><TreeSelect {!isPlatformMode && <Form.Item label={t("users.orgNode")} name="orgId"><TreeSelect
placeholder={t("usersExt.selectOrgPlaceholder")} allowClear treeData={orgTreeData}/></Form.Item>} placeholder={t("usersExt.selectOrgPlaceholder")} allowClear treeData={orgTreeData}/></Form.Item>}
<Row gutter={16}> <Row gutter={16}>

View File

@ -70,6 +70,7 @@ const AiModels: React.FC = () => {
const provider = Form.useWatch("provider", form); const provider = Form.useWatch("provider", form);
const isDefaultChecked = Form.useWatch("isDefaultChecked", form); const isDefaultChecked = Form.useWatch("isDefaultChecked", form);
const isLocalProvider = String(provider || "").toLowerCase() === "custom"; const isLocalProvider = String(provider || "").toLowerCase() === "custom";
const isTencentProvider = String(provider || "").toLowerCase() === "tencent";
const isPlatformAdmin = useMemo(() => { const isPlatformAdmin = useMemo(() => {
const profileStr = sessionStorage.getItem("userProfile"); const profileStr = sessionStorage.getItem("userProfile");
@ -133,11 +134,17 @@ const AiModels: React.FC = () => {
setEditingId(record.id); setEditingId(record.id);
const speakerModel = record.mediaConfig?.speakerModel; const speakerModel = record.mediaConfig?.speakerModel;
const svThreshold = record.mediaConfig?.svThreshold; const svThreshold = record.mediaConfig?.svThreshold;
const tencentAppId = record.mediaConfig?.tencentAppId;
const tencentSecretId = record.mediaConfig?.tencentSecretId;
const tencentSecretKey = record.mediaConfig?.tencentSecretKey;
form.setFieldsValue({ form.setFieldsValue({
...record, ...record,
modelType: record.modelType, modelType: record.modelType,
speakerModel, speakerModel,
svThreshold, svThreshold,
tencentAppId,
tencentSecretId,
tencentSecretKey,
isDefaultChecked: record.isDefault === 1, isDefaultChecked: record.isDefault === 1,
statusChecked: record.status === 1, statusChecked: record.status === 1,
}); });
@ -262,6 +269,12 @@ const AiModels: React.FC = () => {
speakerModel: values.speakerModel, speakerModel: values.speakerModel,
svThreshold: values.svThreshold, svThreshold: values.svThreshold,
} }
: activeType === "ASR" && isTencentProvider
? {
tencentAppId: values.tencentAppId,
tencentSecretId: values.tencentSecretId,
tencentSecretKey: values.tencentSecretKey,
}
: undefined, : undefined,
temperature: values.temperature, temperature: values.temperature,
topP: values.topP, topP: values.topP,
@ -524,16 +537,20 @@ const AiModels: React.FC = () => {
</Col> </Col>
</Row> </Row>
<Form.Item name="baseUrl" label="Base URL" rules={[{ required: true, message: "请输入 Base URL" }]}> {!isTencentProvider && (
<Input placeholder="https://api.example.com" /> <>
</Form.Item> <Form.Item name="baseUrl" label="Base URL" rules={[{required: true, message: "请输入 Base URL"}]}>
<Input placeholder="https://api.example.com"/>
</Form.Item>
<Form.Item <Form.Item
name="apiKey" name="apiKey"
label="API Key" label="API Key"
> >
<Input.Password /> <Input.Password/>
</Form.Item> </Form.Item>
</>
)}
{(activeType === "LLM" || isLocalProvider) && ( {(activeType === "LLM" || isLocalProvider) && (
<Form.Item label="连通性测试"> <Form.Item label="连通性测试">
@ -556,7 +573,10 @@ const AiModels: React.FC = () => {
<Form.Item <Form.Item
name="modelCode" name="modelCode"
noStyle noStyle
rules={activeType === "LLM" ? [{ required: true, message: "请输入或选择模型名称" }] : []} rules={activeType === "LLM" || isTencentProvider ? [{
required: true,
message: "请输入或选择模型名称"
}] : []}
> >
<AutoComplete <AutoComplete
style={{ width: "calc(100% - 100px)" }} style={{ width: "calc(100% - 100px)" }}
@ -574,9 +594,11 @@ const AiModels: React.FC = () => {
<Input allowClear placeholder="可选择或自定义输入模型名称" /> <Input allowClear placeholder="可选择或自定义输入模型名称" />
</AutoComplete> </AutoComplete>
</Form.Item> </Form.Item>
<Button icon={<SyncOutlined spin={fetchLoading} />} onClick={handleFetchRemote} style={{ width: 100 }}> {!isTencentProvider && (
<Button icon={<SyncOutlined spin={fetchLoading}/>} onClick={handleFetchRemote} style={{width: 100}}>
</Button>
</Button>
)}
</Space.Compact> </Space.Compact>
</Form.Item> </Form.Item>
@ -609,6 +631,38 @@ const AiModels: React.FC = () => {
</Row> </Row>
)} )}
{activeType === "ASR" && isTencentProvider && (
<Row gutter={16}>
<Col span={12}>
<Form.Item
name="tencentAppId"
label="App ID"
rules={[{required: true, message: "请输入 App ID"}]}
>
<Input/>
</Form.Item>
</Col>
<Col span={12}>
<Form.Item
name="tencentSecretId"
label="Secret ID"
rules={[{required: true, message: "请输入 Secret ID"}]}
>
<Input/>
</Form.Item>
</Col>
<Col span={24}>
<Form.Item
name="tencentSecretKey"
label="Secret Key"
rules={[{required: true, message: "请输入 Secret Key"}]}
>
<Input.Password/>
</Form.Item>
</Col>
</Row>
)}
{activeType === "LLM" && ( {activeType === "LLM" && (
<> <>
<Form.Item name="apiPath" label="API 路径" initialValue="/v1/chat/completions"> <Form.Item name="apiPath" label="API 路径" initialValue="/v1/chat/completions">

View File

@ -1226,7 +1226,7 @@ const MeetingDetail: React.FC = () => {
const [expandKeywords, setExpandKeywords] = useState(false); const [expandKeywords, setExpandKeywords] = useState(false);
const [expandSummary, setExpandSummary] = useState(false); const [expandSummary, setExpandSummary] = useState(false);
const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]); const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
const [workspaceTab, setWorkspaceTab] = useState<WorkspaceTab>('catalog'); const [workspaceTab, setWorkspaceTab] = useState<WorkspaceTab>('transcript');
const [addingHotwords, setAddingHotwords] = useState(false); const [addingHotwords, setAddingHotwords] = useState(false);
const [editingTranscriptId, setEditingTranscriptId] = useState<number | null>(null); const [editingTranscriptId, setEditingTranscriptId] = useState<number | null>(null);
const [savingTranscriptId, setSavingTranscriptId] = useState<number | null>(null); const [savingTranscriptId, setSavingTranscriptId] = useState<number | null>(null);
@ -1402,15 +1402,16 @@ const MeetingDetail: React.FC = () => {
return false; return false;
}, [meeting]); }, [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const canRetrySummary = isOwner const canRetrySummary = isOwner
&& transcripts.length > 0 && transcripts.length > 0
&& meeting?.status !== 1 && meeting?.status !== 1
&& meeting?.status !== 2 && meeting?.status !== 2
&& meeting?.latestSummaryAttemptStatus !== 3 && meeting?.latestSummaryAttemptStatus !== 3
&& meeting?.latestChapterAttemptStatus !== 3; && (!aiCatalogEnabled || meeting?.latestChapterAttemptStatus !== 3);
const canRetryTranscription = isOwner && meeting?.status === 4 && transcripts.length === 0 && !!meeting?.audioUrl; const canRetryTranscription = isOwner && meeting?.status === 4 && transcripts.length === 0 && !!meeting?.audioUrl;
const canRetryFailedSummaryTask = isOwner && meeting?.latestSummaryAttemptStatus === 3 && meeting?.status !== 2; const canRetryFailedSummaryTask = isOwner && meeting?.latestSummaryAttemptStatus === 3 && meeting?.status !== 2;
const canRetryFailedChapterTask = isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2; const canRetryFailedChapterTask = aiCatalogEnabled && isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2;
const canRetrySchedule = isOwner && meeting?.status === 0 && (!generationProgress || generationProgress.percent <= 0) && !!generationProgress?.queuedAt && dayjs().diff(dayjs(generationProgress.queuedAt)) >= QUEUED_RETRY_THRESHOLD_MS; const canRetrySchedule = isOwner && meeting?.status === 0 && (!generationProgress || generationProgress.percent <= 0) && !!generationProgress?.queuedAt && dayjs().diff(dayjs(generationProgress.queuedAt)) >= QUEUED_RETRY_THRESHOLD_MS;
@ -1433,14 +1434,14 @@ const MeetingDetail: React.FC = () => {
}; };
const hasSummaryContent = Boolean(meeting?.summaryContent?.trim()); const hasSummaryContent = Boolean(meeting?.summaryContent?.trim());
const hasCatalogContent = catalogChapterLinks.length > 0; const hasCatalogContent = aiCatalogEnabled && catalogChapterLinks.length > 0;
const generationFailureNotice = useMemo<MeetingStateNotice | null>(() => { const generationFailureNotice = useMemo<MeetingStateNotice | null>(() => {
if (!meeting || meeting.status !== 4) { if (!meeting || meeting.status !== 4) {
return null; return null;
} }
const hasFallbackContent = hasSummaryContent || hasCatalogContent; const hasFallbackContent = hasSummaryContent || hasCatalogContent;
if (meeting.latestChapterAttemptStatus === 3) { if (aiCatalogEnabled && meeting.latestChapterAttemptStatus === 3) {
const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败'; const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败';
return { return {
title: hasFallbackContent ? '历史内容仍可查看' : '本次 AI 目录生成失败', title: hasFallbackContent ? '历史内容仍可查看' : '本次 AI 目录生成失败',
@ -1467,45 +1468,13 @@ const MeetingDetail: React.FC = () => {
} }
return { return {
title: hasFallbackContent ? '历史内容仍可查看' : '会议处理异常', title: '会议处理异常',
description: hasFallbackContent description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。',
? '最近一次处理未成功,当前展示的是最近一次成功生成的内容。你可以继续查看,或重新发起识别/总结。'
: '会议在处理中遇到问题,暂时没有可展示的总结内容。你可以重新发起识别或总结。',
type: hasFallbackContent ? 'info' : 'warning', type: hasFallbackContent ? 'info' : 'warning',
hasFallbackContent, hasFallbackContent,
scope: 'global', scope: 'global',
}; };
if (meeting.latestChapterAttemptStatus === 3) { }, [aiCatalogEnabled, hasCatalogContent, hasSummaryContent, meeting]);
const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败';
return {
key: `chapter-${meeting.latestChapterAttemptTaskId ?? 'latest'}`,
title: '本次重新总结失败',
description: hasFallbackContent
? `章节生成失败,当前展示的是上一次成功的摘要和 AI 目录。失败原因:${detail}`
: `章节生成失败,且当前没有可展示的历史摘要或 AI 目录。失败原因:${detail}`,
hasFallbackContent,
};
}
if (meeting.latestSummaryAttemptStatus === 3) {
const detail = meeting.latestSummaryAttemptErrorMsg || '总结生成失败';
return {
key: `summary-${meeting.latestSummaryAttemptTaskId ?? 'latest'}`,
title: '本次重新总结失败',
description: hasFallbackContent
? `总结生成失败,当前展示的是上一次成功的摘要和 AI 目录。失败原因:${detail}`
: `总结生成失败,且当前没有可展示的历史摘要或 AI 目录。失败原因:${detail}`,
hasFallbackContent,
};
}
return {
key: 'general-failure',
title: '会议处理异常',
description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。',
hasFallbackContent,
};
}, [hasCatalogContent, hasSummaryContent, meeting]);
const summaryPanelNotice = useMemo<MeetingStateNotice | null>(() => { const summaryPanelNotice = useMemo<MeetingStateNotice | null>(() => {
if (!meeting || !hasSummaryContent) { if (!meeting || !hasSummaryContent) {
return null; return null;
@ -1522,7 +1491,7 @@ const MeetingDetail: React.FC = () => {
return null; return null;
}, [generationFailureNotice, hasSummaryContent, meeting]); }, [generationFailureNotice, hasSummaryContent, meeting]);
const catalogPanelNotice = useMemo<MeetingStateNotice | null>(() => { const catalogPanelNotice = useMemo<MeetingStateNotice | null>(() => {
if (!generationFailureNotice || generationFailureNotice.scope !== 'catalog') { if (!aiCatalogEnabled || !generationFailureNotice || generationFailureNotice.scope !== 'catalog') {
return null; return null;
} }
if (generationFailureNotice.hasFallbackContent && hasCatalogContent) { if (generationFailureNotice.hasFallbackContent && hasCatalogContent) {
@ -1535,7 +1504,7 @@ const MeetingDetail: React.FC = () => {
}; };
} }
return generationFailureNotice; return generationFailureNotice;
}, [generationFailureNotice, hasCatalogContent]); }, [aiCatalogEnabled, generationFailureNotice, hasCatalogContent]);
const emptyTranscriptFailureNotice = useMemo(() => { const emptyTranscriptFailureNotice = useMemo(() => {
if (!meeting || meeting.status !== 4 || transcripts.length > 0) { if (!meeting || meeting.status !== 4 || transcripts.length > 0) {
return null; return null;
@ -1557,12 +1526,18 @@ const MeetingDetail: React.FC = () => {
} }
}, [meeting?.id, meeting?.status]); }, [meeting?.id, meeting?.status]);
useEffect(() => {
if (!aiCatalogEnabled && workspaceTab === 'catalog') {
setWorkspaceTab('transcript');
}
}, [aiCatalogEnabled, workspaceTab]);
useEffect(() => { useEffect(() => {
const attemptKey = String(meeting?.latestChapterAttemptTaskId ?? meeting?.latestSummaryAttemptTaskId ?? ''); const attemptKey = String(meeting?.latestChapterAttemptTaskId ?? meeting?.latestSummaryAttemptTaskId ?? '');
if (meeting?.status !== 2 || !attemptKey) { if (meeting?.status !== 2 || !attemptKey) {
return; return;
} }
if ((generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) { if (!aiCatalogEnabled || (generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) {
return; return;
} }
if (autoOpenedCatalogAttemptRef.current === attemptKey) { if (autoOpenedCatalogAttemptRef.current === attemptKey) {
@ -1571,6 +1546,7 @@ const MeetingDetail: React.FC = () => {
autoOpenedCatalogAttemptRef.current = attemptKey; autoOpenedCatalogAttemptRef.current = attemptKey;
setWorkspaceTab('catalog'); setWorkspaceTab('catalog');
}, [ }, [
aiCatalogEnabled,
catalogChapterLinks.length, catalogChapterLinks.length,
generationProgress?.percent, generationProgress?.percent,
meeting?.latestChapterAttemptTaskId, meeting?.latestChapterAttemptTaskId,
@ -2766,13 +2742,15 @@ const MeetingDetail: React.FC = () => {
)} )}
<div className="transcript-stage-tabs"> <div className="transcript-stage-tabs">
<button {aiCatalogEnabled && (
type="button" <button
className={workspaceTab === 'catalog' ? 'active' : ''} type="button"
onClick={() => setWorkspaceTab('catalog')} className={workspaceTab === 'catalog' ? 'active' : ''}
> onClick={() => setWorkspaceTab('catalog')}
AI >
</button> AI
</button>
)}
<button <button
type="button" type="button"
className={workspaceTab === 'transcript' ? 'active' : ''} className={workspaceTab === 'transcript' ? 'active' : ''}
@ -2783,7 +2761,7 @@ const MeetingDetail: React.FC = () => {
</div> </div>
<div className="transcript-scroll-shell"> <div className="transcript-scroll-shell">
{workspaceTab === 'catalog' ? ( {aiCatalogEnabled && workspaceTab === 'catalog' ? (
<div className="catalog-list"> <div className="catalog-list">
{catalogPanelNotice && ( {catalogPanelNotice && (
<Alert <Alert

View File

@ -339,6 +339,7 @@ export default function MeetingPreview() {
const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]); const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]);
const keywords = useMemo(() => analysis.keywords || [], [analysis.keywords]); const keywords = useMemo(() => analysis.keywords || [], [analysis.keywords]);
const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]); const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const statusMeta = STATUS_META[meeting?.status || 0] || { const statusMeta = STATUS_META[meeting?.status || 0] || {
label: TEXT.statusPending, label: TEXT.statusPending,
className: "is-warning", className: "is-warning",
@ -416,6 +417,12 @@ export default function MeetingPreview() {
}); });
}, [analysis.chapters, meetingChapters, transcripts]); }, [analysis.chapters, meetingChapters, transcripts]);
useEffect(() => {
if (!aiCatalogEnabled && pageTab === "catalog") {
setPageTab("summary");
}
}, [aiCatalogEnabled, pageTab]);
useEffect(() => { useEffect(() => {
if (!activeTranscriptId) { if (!activeTranscriptId) {
return; return;
@ -967,7 +974,7 @@ export default function MeetingPreview() {
onChange={(key) => setPageTab(key as PreviewPageTab)} onChange={(key) => setPageTab(key as PreviewPageTab)}
items={[ items={[
{ key: "summary", label: TEXT.pageSummary }, { key: "summary", label: TEXT.pageSummary },
{ key: "catalog", label: TEXT.pageCatalog }, ...(aiCatalogEnabled ? [{ key: "catalog", label: TEXT.pageCatalog }] : []),
{ key: "transcript", label: TEXT.pageTranscript }, { key: "transcript", label: TEXT.pageTranscript },
]} ]}
/> />
@ -975,7 +982,7 @@ export default function MeetingPreview() {
<div className="meeting-preview-tab-content"> <div className="meeting-preview-tab-content">
{pageTab === "summary" ? summaryTabContent : null} {pageTab === "summary" ? summaryTabContent : null}
{pageTab === "catalog" ? catalogTabContent : null} {aiCatalogEnabled && pageTab === "catalog" ? catalogTabContent : null}
{pageTab === "transcript" ? transcriptTabContent : null} {pageTab === "transcript" ? transcriptTabContent : null}
</div> </div>
</div> </div>

View File

@ -10,7 +10,6 @@
InfoCircleOutlined, InfoCircleOutlined,
PauseCircleOutlined, PauseCircleOutlined,
PlusOutlined, PlusOutlined,
RightOutlined,
SearchOutlined, SearchOutlined,
SettingOutlined, SettingOutlined,
SyncOutlined, SyncOutlined,
@ -136,6 +135,9 @@ const shouldPollMeetingCard = (item: MeetingVO) =>
const getUnifiedStatusCode = (progress: MeetingProgress | null | undefined) => const getUnifiedStatusCode = (progress: MeetingProgress | null | undefined) =>
progress?.unifiedStatus?.statusCode; progress?.unifiedStatus?.statusCode;
const isWaitingUploadProgress = (progress: MeetingProgress | null | undefined) =>
getUnifiedStatusCode(progress) === "WAITING_UPLOAD";
const getEffectiveStatus = (item: MeetingVO, progress: MeetingProgress | null) => { const getEffectiveStatus = (item: MeetingVO, progress: MeetingProgress | null) => {
const unifiedStatusCode = getUnifiedStatusCode(progress); const unifiedStatusCode = getUnifiedStatusCode(progress);
if (unifiedStatusCode === "WAITING_UPLOAD") { if (unifiedStatusCode === "WAITING_UPLOAD") {
@ -285,12 +287,17 @@ const MeetingCardItem: React.FC<{
}> = ({ item, config, progress, onOpenMeeting, onRetrySchedule, onDelete, retrying }) => { }> = ({ item, config, progress, onOpenMeeting, onRetrySchedule, onDelete, retrying }) => {
const effectiveStatus = getEffectiveStatus(item, progress); const effectiveStatus = getEffectiveStatus(item, progress);
const isProcessing = shouldTrackGenerationProgress(item); const isProcessing = shouldTrackGenerationProgress(item);
const isWaitingUpload = isWaitingUploadProgress(progress) || effectiveStatus === 8;
const isPaused = effectiveStatus === PAUSED_DISPLAY_STATUS; const isPaused = effectiveStatus === PAUSED_DISPLAY_STATUS;
const isRealtimeActive = effectiveStatus === REALTIME_ACTIVE_DISPLAY_STATUS; const isRealtimeActive = effectiveStatus === REALTIME_ACTIVE_DISPLAY_STATUS;
const isRealtimeIdle = effectiveStatus === REALTIME_IDLE_DISPLAY_STATUS; const isRealtimeIdle = effectiveStatus === REALTIME_IDLE_DISPLAY_STATUS;
const isCrossPlatformRealtime = (isPaused || isRealtimeActive || isRealtimeIdle) && !canControlRealtimeFromCurrentPlatform(item); const isCrossPlatformRealtime = (isPaused || isRealtimeActive || isRealtimeIdle) && !canControlRealtimeFromCurrentPlatform(item);
const crossPlatformHint = `${getRealtimeSourceLabel(item)} 继续`; const crossPlatformHint = `${getRealtimeSourceLabel(item)} 继续`;
const canRetry = canRetryQueuedMeeting(item, progress); const canRetry = canRetryQueuedMeeting(item, progress);
const ownerName = item.creatorName || "未知";
const processingMessage = isWaitingUpload
? (progress?.message || progress?.unifiedStatus?.message || config.text)
: (progress?.unifiedStatus?.message || progress?.message || "深度分析中...");
const sourceColor = item.meetingSource === "ANDROID" ? "#10b981" : "#3b82f6"; const sourceColor = item.meetingSource === "ANDROID" ? "#10b981" : "#3b82f6";
@ -375,26 +382,28 @@ const MeetingCardItem: React.FC<{
fontSize: "12px" fontSize: "12px"
}} }}
> >
<div style={{ display: "flex", alignItems: "center", gap: "6px", color: config.color, fontWeight: 600 }}> <div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: "10px" }}>
{isProcessing ? <SyncOutlined spin /> : <InfoCircleOutlined />} <div style={{ display: "flex", alignItems: "center", gap: "6px", minWidth: 0, color: config.color, fontWeight: 600 }}>
<span style={{ fontSize: "12px" }}> {isProcessing ? <SyncOutlined spin /> : <InfoCircleOutlined />}
{isProcessing ? (progress?.message || "深度分析中...") : (isCrossPlatformRealtime ? crossPlatformHint : config.text)} <span style={{ fontSize: "12px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
</span> {isProcessing ? processingMessage : (isCrossPlatformRealtime ? crossPlatformHint : config.text)}
</span>
</div>
{canRetry && (
<Button
type="link"
size="small"
loading={retrying}
onClick={(event) => {
event.stopPropagation();
onRetrySchedule(item);
}}
style={{ paddingInline: 0, height: 18, lineHeight: "18px", flexShrink: 0, fontSize: "12px", fontWeight: 600 }}
>
</Button>
)}
</div> </div>
{canRetry && (
<Button
type="link"
size="small"
loading={retrying}
onClick={(event) => {
event.stopPropagation();
onRetrySchedule(item);
}}
style={{ paddingInline: 0, height: "auto" }}
>
</Button>
)}
</div> </div>
) : ( ) : (
<div style={{ display: "flex", flexWrap: "wrap", gap: "12px", color: "#8c8c8c", fontSize: "12px" }}> <div style={{ display: "flex", flexWrap: "wrap", gap: "12px", color: "#8c8c8c", fontSize: "12px" }}>
@ -402,10 +411,6 @@ const MeetingCardItem: React.FC<{
<CalendarOutlined style={{ fontSize: "13px", opacity: 0.7 }} /> <CalendarOutlined style={{ fontSize: "13px", opacity: 0.7 }} />
<span>{dayjs(item.meetingTime).format("MM-DD HH:mm")}</span> <span>{dayjs(item.meetingTime).format("MM-DD HH:mm")}</span>
</div> </div>
<div style={{ display: "flex", alignItems: "center", gap: "4px" }}>
<UserOutlined style={{ fontSize: "13px", opacity: 0.7 }} />
<span style={{ maxWidth: "80px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{item.creatorName || "未知"}</span>
</div>
</div> </div>
)} )}
</div> </div>
@ -436,31 +441,27 @@ const MeetingCardItem: React.FC<{
<div <div
style={{ style={{
height: "28px", height: "28px",
maxWidth: "112px",
padding: "0 10px",
borderRadius: "8px", borderRadius: "8px",
background: "rgba(95, 81, 255, 0.1)", background: "rgba(15, 23, 42, 0.04)",
color: "var(--primary-blue)", color: "#6b7280",
display: "flex", display: "flex",
alignItems: "center", alignItems: "center",
justifyContent: "center", justifyContent: "center",
transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)", gap: "5px",
padding: "0 10px",
overflow: "hidden", overflow: "hidden",
whiteSpace: "nowrap", whiteSpace: "nowrap",
minWidth: "28px" flexShrink: 0,
}}
className="card-arrow"
>
<RightOutlined style={{ fontSize: "10px", flexShrink: 0 }} />
<span className="view-detail-text" style={{
fontSize: "11px", fontSize: "11px",
fontWeight: 700, fontWeight: 600
maxWidth: 0, }}
opacity: 0, title={`所属用户:${ownerName}`}
transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)", >
marginLeft: 0, <UserOutlined style={{ fontSize: "12px", opacity: 0.75, flexShrink: 0 }} />
overflow: 'hidden', <span style={{ overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
display: 'inline-block' {ownerName}
}}></span> </span>
</div> </div>
</div> </div>
</Card> </Card>
@ -1001,8 +1002,6 @@ const Meetings: React.FC = () => {
<style>{` <style>{`
.meeting-card-v2:hover { transform: translateY(-3px); box-shadow: 0 12px 24px rgba(95, 81, 255, 0.08) !important; border-color: rgba(95, 81, 255, 0.15) !important; } .meeting-card-v2:hover { transform: translateY(-3px); box-shadow: 0 12px 24px rgba(95, 81, 255, 0.08) !important; border-color: rgba(95, 81, 255, 0.15) !important; }
.meeting-card-v2:hover .card-arrow { background: var(--primary-blue) !important; color: white !important; padding: 0 12px; }
.meeting-card-v2:hover .view-detail-text { max-width: 60px; opacity: 1; margin-left: 6px; }
.status-bar-active { animation: statusBreathing 2s infinite ease-in-out; } .status-bar-active { animation: statusBreathing 2s infinite ease-in-out; }
@keyframes statusBreathing { 0% { opacity: 1; } 50% { opacity: 0.4; } 100% { opacity: 1; } } @keyframes statusBreathing { 0% { opacity: 1; } 50% { opacity: 0.4; } 100% { opacity: 1; } }
.meetings-list-table.ant-table-wrapper, .meetings-list-table.ant-table-wrapper,

View File

@ -18,7 +18,6 @@ import dayjs from "dayjs";
import PageHeader from "../../components/shared/PageHeader"; import PageHeader from "../../components/shared/PageHeader";
import PageContainer from "@/components/shared/PageContainer"; import PageContainer from "@/components/shared/PageContainer";
import { import {
appendRealtimeTranscripts,
completeRealtimeMeeting, completeRealtimeMeeting,
getMeetingDetail, getMeetingDetail,
getRealtimeMeetingSessionStatus, getRealtimeMeetingSessionStatus,
@ -28,7 +27,6 @@ import {
type MeetingTranscriptVO, type MeetingTranscriptVO,
type MeetingVO, type MeetingVO,
type RealtimeMeetingSessionStatus, type RealtimeMeetingSessionStatus,
type RealtimeTranscriptItemDTO,
type RealtimeSocketSessionVO, type RealtimeSocketSessionVO,
} from "../../api/business/meeting"; } from "../../api/business/meeting";
const { Text, Title } = Typography; const { Text, Title } = Typography;
@ -44,6 +42,8 @@ type WsMessage = {
data?: { data?: {
text?: string; text?: string;
is_final?: boolean; is_final?: boolean;
sentence_id?: number;
sentence_key?: string;
start?: number; start?: number;
end?: number; end?: number;
speaker_id?: string; speaker_id?: string;
@ -58,6 +58,8 @@ type WsMessage = {
type TranscriptCard = { type TranscriptCard = {
id: string; id: string;
sentenceKey?: string;
sentenceId?: number;
speakerName: string; speakerName: string;
userId?: string | number; userId?: string | number;
text: string; text: string;
@ -66,6 +68,16 @@ type TranscriptCard = {
final: boolean; final: boolean;
}; };
type NormalizedWsMessage = {
sentenceKey?: string;
text: string;
isFinal: boolean;
sentenceId?: number;
speaker?: WsSpeaker;
startTime?: number;
endTime?: number;
};
type RealtimeMeetingSessionDraft = { type RealtimeMeetingSessionDraft = {
meetingId: number; meetingId: number;
meetingTitle: string; meetingTitle: string;
@ -123,12 +135,18 @@ function resolveSpeaker(speaker?: WsSpeaker) {
return { speakerId: "spk_0", speakerName: "Unknown", userId: undefined }; return { speakerId: "spk_0", speakerName: "Unknown", userId: undefined };
} }
if (typeof speaker === "string") { if (typeof speaker === "string") {
return { speakerId: speaker, speakerName: speaker, userId: undefined }; const normalized = speaker.trim();
if (!normalized || normalized === "-1") {
return {speakerId: "spk_0", speakerName: "Unknown", userId: undefined};
}
return {speakerId: normalized, speakerName: normalized, userId: undefined};
} }
const rawUserId = speaker.user_id === null || speaker.user_id === undefined ? undefined : String(speaker.user_id).trim();
const speakerId = rawUserId && rawUserId !== "-1" ? rawUserId : "spk_0";
return { return {
speakerId: speaker.user_id ? String(speaker.user_id) : "spk_0", speakerId,
speakerName: speaker.name || (speaker.user_id ? String(speaker.user_id) : "Unknown"), speakerName: speaker.name?.trim() || (speakerId !== "spk_0" ? speakerId : "Unknown"),
userId: speaker.user_id, userId: speakerId === "spk_0" ? undefined : rawUserId,
}; };
} }
@ -159,17 +177,29 @@ function toMs(value?: number) {
return Math.round(value * 1000); return Math.round(value * 1000);
} }
function buildTranscriptCardId(sentenceKey?: string, sentenceId?: number) {
if (sentenceKey) {
return sentenceKey;
}
if (sentenceId === undefined || sentenceId === null) {
return `live-${Date.now()}-${Math.random()}`;
}
return `sentence-${sentenceId}`;
}
function buildRealtimeProxyWsUrl(socketSession: RealtimeSocketSessionVO) { function buildRealtimeProxyWsUrl(socketSession: RealtimeSocketSessionVO) {
const protocol = window.location.protocol === "https:" ? "wss" : "ws"; const protocol = window.location.protocol === "https:" ? "wss" : "ws";
return `${protocol}://${window.location.host}${socketSession.path}?sessionToken=${encodeURIComponent(socketSession.sessionToken)}`; return `${protocol}://${window.location.host}${socketSession.path}?sessionToken=${encodeURIComponent(socketSession.sessionToken)}`;
} }
function normalizeWsMessage(payload: WsMessage) { function normalizeWsMessage(payload: WsMessage): NormalizedWsMessage | null {
if (payload.type === "partial" || payload.type === "segment") { if (payload.type === "partial" || payload.type === "segment") {
const data = payload.data || {}; const data = payload.data || {};
return { return {
text: data.text || "", text: data.text || "",
isFinal: payload.type === "segment" || !!data.is_final, isFinal: payload.type === "segment" || !!data.is_final,
sentenceKey: data.sentence_key,
sentenceId: data.sentence_id,
speaker: { speaker: {
name: data.speaker_name, name: data.speaker_name,
user_id: data.user_id ?? data.speaker_id, user_id: data.user_id ?? data.speaker_id,
@ -212,7 +242,6 @@ export function RealtimeAsrSession() {
const [audioLevel, setAudioLevel] = useState(0); const [audioLevel, setAudioLevel] = useState(0);
const [elapsedSeconds, setElapsedSeconds] = useState(0); const [elapsedSeconds, setElapsedSeconds] = useState(0);
const [sessionStatus, setSessionStatus] = useState<RealtimeMeetingSessionStatus | null>(null); const [sessionStatus, setSessionStatus] = useState<RealtimeMeetingSessionStatus | null>(null);
const transcriptRef = useRef<HTMLDivElement | null>(null); const transcriptRef = useRef<HTMLDivElement | null>(null);
const wsRef = useRef<WebSocket | null>(null); const wsRef = useRef<WebSocket | null>(null);
const audioContextRef = useRef<AudioContext | null>(null); const audioContextRef = useRef<AudioContext | null>(null);
@ -223,6 +252,7 @@ export function RealtimeAsrSession() {
const completeOnceRef = useRef(false); const completeOnceRef = useRef(false);
const startedAtRef = useRef<number | null>(null); const startedAtRef = useRef<number | null>(null);
const sessionStartedRef = useRef(false); const sessionStartedRef = useRef(false);
const elapsedOffsetRef = useRef(0);
const finalTranscriptCount = transcripts.length; const finalTranscriptCount = transcripts.length;
const totalTranscriptChars = useMemo( const totalTranscriptChars = useMemo(
@ -281,7 +311,7 @@ export function RealtimeAsrSession() {
} }
setTranscripts( setTranscripts(
(transcriptRes.data.data || []).map((item: MeetingTranscriptVO) => ({ (transcriptRes.data.data || []).map((item: MeetingTranscriptVO) => ({
id: String(item.id), id: `persisted-${item.id}`,
speakerName: item.speakerName || item.speakerId || "发言人", speakerName: item.speakerName || item.speakerId || "发言人",
text: item.content, text: item.content,
startTime: item.startTime, startTime: item.startTime,
@ -301,12 +331,12 @@ export function RealtimeAsrSession() {
useEffect(() => { useEffect(() => {
if (!recording) { if (!recording) {
setElapsedSeconds(0); setElapsedSeconds(elapsedOffsetRef.current);
return; return;
} }
const timer = window.setInterval(() => { const timer = window.setInterval(() => {
if (startedAtRef.current) { if (startedAtRef.current) {
setElapsedSeconds(Math.floor((Date.now() - startedAtRef.current) / 1000)); setElapsedSeconds(elapsedOffsetRef.current + Math.floor((Date.now() - startedAtRef.current) / 1000));
} }
}, 1000); }, 1000);
return () => window.clearInterval(timer); return () => window.clearInterval(timer);
@ -326,7 +356,7 @@ export function RealtimeAsrSession() {
} }
const token = localStorage.getItem("accessToken"); const token = localStorage.getItem("accessToken");
if (wsRef.current?.readyState === WebSocket.OPEN) { if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ is_speaking: false })); wsRef.current.close();
} }
fetch(`/api/biz/meeting/${meetingId}/realtime/pause`, { fetch(`/api/biz/meeting/${meetingId}/realtime/pause`, {
method: "POST", method: "POST",
@ -360,6 +390,44 @@ export function RealtimeAsrSession() {
setAudioLevel(0); setAudioLevel(0);
}; };
const closeFrontendSocket = async (sendStop: boolean) => {
const socket = wsRef.current;
if (sendStop && socket?.readyState === WebSocket.OPEN) {
socket.send(JSON.stringify({type: "stop"}));
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
socket?.close();
wsRef.current = null;
sessionStartedRef.current = false;
};
const upsertTranscriptCard = (normalized: NormalizedWsMessage, speaker: ReturnType<typeof resolveSpeaker>) => {
setTranscripts((prev) => {
const next = [...prev];
const cardId = buildTranscriptCardId(normalized.sentenceKey, normalized.sentenceId);
const nextCard: TranscriptCard = {
id: cardId,
sentenceKey: normalized.sentenceKey,
sentenceId: normalized.sentenceId,
speakerName: speaker.speakerName,
userId: speaker.userId,
text: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
final: true,
};
if (normalized.sentenceKey || (normalized.sentenceId !== undefined && normalized.sentenceId !== null)) {
const index = next.findIndex((item) => item.id === cardId);
if (index >= 0) {
next[index] = {...next[index], ...nextCard};
return next;
}
}
next.push(nextCard);
return next;
});
};
const handleFatalRealtimeError = async (errorMessage: string) => { const handleFatalRealtimeError = async (errorMessage: string) => {
setConnecting(false); setConnecting(false);
setRecording(false); setRecording(false);
@ -369,6 +437,8 @@ export function RealtimeAsrSession() {
wsRef.current = null; wsRef.current = null;
await shutdownAudioPipeline(); await shutdownAudioPipeline();
startedAtRef.current = null; startedAtRef.current = null;
elapsedOffsetRef.current = 0;
setElapsedSeconds(0);
message.error(errorMessage); message.error(errorMessage);
}; };
@ -419,26 +489,6 @@ export function RealtimeAsrSession() {
processor.connect(audioContext.destination); processor.connect(audioContext.destination);
}; };
const saveFinalTranscript = async (normalized: {
text: string;
speaker?: WsSpeaker;
startTime?: number;
endTime?: number;
}) => {
if (!normalized.text || !meetingId) {
return;
}
const speaker = resolveSpeaker(normalized.speaker);
const item: RealtimeTranscriptItemDTO = {
speakerId: speaker.speakerId,
speakerName: speaker.speakerName,
content: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
};
await appendRealtimeTranscripts(meetingId, [item]);
};
const handlePause = async () => { const handlePause = async () => {
if (!meetingId || pausing || finishing || (!recording && !connecting)) { if (!meetingId || pausing || finishing || (!recording && !connecting)) {
return; return;
@ -451,20 +501,19 @@ export function RealtimeAsrSession() {
setPausing(true); setPausing(true);
setStatusText("暂停识别中..."); setStatusText("暂停识别中...");
try { try {
if (wsRef.current?.readyState === WebSocket.OPEN) { if (recording && startedAtRef.current) {
wsRef.current.send(JSON.stringify({ is_speaking: false })); elapsedOffsetRef.current += Math.floor((Date.now() - startedAtRef.current) / 1000);
} }
wsRef.current?.close();
wsRef.current = null;
sessionStartedRef.current = false;
await shutdownAudioPipeline();
const pauseRes = await pauseRealtimeMeeting(meetingId); const pauseRes = await pauseRealtimeMeeting(meetingId);
await closeFrontendSocket(false);
await shutdownAudioPipeline();
setSessionStatus(pauseRes.data.data); setSessionStatus(pauseRes.data.data);
setRecording(false); setRecording(false);
setConnecting(false); setConnecting(false);
startedAtRef.current = null; startedAtRef.current = null;
setStatusText(pauseRes.data.data?.hasTranscript ? "已暂停,可继续识别" : "已暂停,当前还没有转录内容"); setElapsedSeconds(elapsedOffsetRef.current);
message.success("实时识别已暂停"); setStatusText("已暂停,可继续会议并等待说话人修正");
message.success("实时会议已暂停");
} catch (error) { } catch (error) {
setStatusText("暂停失败"); setStatusText("暂停失败");
message.error(error instanceof Error ? error.message : "暂停实时识别失败"); message.error(error instanceof Error ? error.message : "暂停实时识别失败");
@ -547,21 +596,9 @@ export function RealtimeAsrSession() {
const speaker = resolveSpeaker(normalized.speaker); const speaker = resolveSpeaker(normalized.speaker);
if (normalized.isFinal) { if (normalized.isFinal) {
setTranscripts((prev) => [ upsertTranscriptCard(normalized, speaker);
...prev,
{
id: `${Date.now()}-${Math.random()}`,
speakerName: speaker.speakerName,
userId: speaker.userId,
text: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
final: true,
},
]);
setStreamingText(""); setStreamingText("");
setStreamingSpeaker("Unknown"); setStreamingSpeaker("Unknown");
void saveFinalTranscript(normalized);
} else { } else {
setStreamingText(normalized.text); setStreamingText(normalized.text);
setStreamingSpeaker(speaker.speakerName); setStreamingSpeaker(speaker.speakerName);
@ -602,12 +639,7 @@ export function RealtimeAsrSession() {
setFinishing(true); setFinishing(true);
setStatusText("结束会议中..."); setStatusText("结束会议中...");
if (wsRef.current?.readyState === WebSocket.OPEN) { await closeFrontendSocket(true);
wsRef.current.send(JSON.stringify({ is_speaking: false }));
}
wsRef.current?.close();
wsRef.current = null;
sessionStartedRef.current = false;
await shutdownAudioPipeline(); await shutdownAudioPipeline();

View File

@ -7,9 +7,10 @@ interface PageHeaderProps {
title: string; title: string;
extra?: ReactNode; extra?: ReactNode;
back?: boolean; back?: boolean;
onBack?: () => void;
} }
export default function PageHeader({ title, extra, back = false }: PageHeaderProps) { export default function PageHeader({ title, extra, back = false, onBack }: PageHeaderProps) {
const navigate = useNavigate(); const navigate = useNavigate();
return ( return (
@ -20,7 +21,7 @@ export default function PageHeader({ title, extra, back = false }: PageHeaderPro
type="text" type="text"
shape="circle" shape="circle"
icon={<ArrowLeftOutlined />} icon={<ArrowLeftOutlined />}
onClick={() => navigate(-1)} onClick={() => (onBack ? onBack() : navigate(-1))}
className="page-header__back" className="page-header__back"
/> />
) : null} ) : null}

View File

@ -1,4 +1,4 @@
import {useMemo, useRef, useState} from "react"; import {useEffect, useMemo, useRef, useState} from "react";
import { import {
AudioOutlined, AudioOutlined,
CalendarOutlined, CalendarOutlined,
@ -195,6 +195,7 @@ export default function MeetingPreviewView({
}, [meeting?.participants, transcripts]); }, [meeting?.participants, transcripts]);
const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]); const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]);
const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]); const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const statusMeta = STATUS_META[meeting?.status || 0] || { const statusMeta = STATUS_META[meeting?.status || 0] || {
label: TEXT.statusPending, label: TEXT.statusPending,
className: "is-warning", className: "is-warning",
@ -267,6 +268,12 @@ export default function MeetingPreviewView({
}); });
}, [analysis.chapters, meetingChapters, transcripts]); }, [analysis.chapters, meetingChapters, transcripts]);
useEffect(() => {
if (!aiCatalogEnabled && pageTab === "catalog") {
setPageTab("summary");
}
}, [aiCatalogEnabled, pageTab]);
const handleTranscriptSeek = (item: MeetingTranscriptVO) => { const handleTranscriptSeek = (item: MeetingTranscriptVO) => {
if (!audioRef.current) return; if (!audioRef.current) return;
audioRef.current.currentTime = Math.max(0, (item.startTime || 0) / 1000); audioRef.current.currentTime = Math.max(0, (item.startTime || 0) / 1000);
@ -459,7 +466,7 @@ export default function MeetingPreviewView({
onChange={(key) => setPageTab(key as PreviewPageTab)} onChange={(key) => setPageTab(key as PreviewPageTab)}
items={[ items={[
{key: "summary", label: TEXT.pageSummary}, {key: "summary", label: TEXT.pageSummary},
{key: "catalog", label: TEXT.pageCatalog}, ...(aiCatalogEnabled ? [{key: "catalog", label: TEXT.pageCatalog}] : []),
{key: "transcript", label: TEXT.pageTranscript}, {key: "transcript", label: TEXT.pageTranscript},
]} ]}
/> />
@ -489,7 +496,7 @@ export default function MeetingPreviewView({
</> </>
) : null} ) : null}
{pageTab === "catalog" ? ( {aiCatalogEnabled && pageTab === "catalog" ? (
<div className="meeting-preview-catalog-list"> <div className="meeting-preview-catalog-list">
{catalogChapterLinks.length ? ( {catalogChapterLinks.length ? (
catalogChapterLinks.map((chapter, index) => ( catalogChapterLinks.map((chapter, index) => (

View File

@ -1,18 +1,21 @@
import { App, Button, Card, Input, Space, Typography } from "antd"; import { App, Button, Card, Input, Space, Typography } from "antd";
import { useEffect, useMemo, useState } from "react"; import { useEffect, useMemo, useState } from "react";
import { useParams, useSearchParams } from "react-router-dom"; import { useNavigate, useParams, useSearchParams } from "react-router-dom";
import { getMeetingPreviewAccess, getPublicMeetingPreview } from "@/api/meeting"; import { getMeetingPreviewAccess, getPublicMeetingPreview } from "@/api/meeting";
import LoadingScreen from "@/components/LoadingScreen"; import LoadingScreen from "@/components/LoadingScreen";
import PageHeader from "@/components/PageHeader";
import MeetingPreviewView from "@/components/preview/MeetingPreviewView"; import MeetingPreviewView from "@/components/preview/MeetingPreviewView";
import usePageTitle from "@/hooks/usePageTitle"; import usePageTitle from "@/hooks/usePageTitle";
import type { MeetingChapterVO, MeetingTranscriptVO, MeetingVO } from "@/types"; import type { MeetingChapterVO, MeetingTranscriptVO, MeetingVO } from "@/types";
import { hasAccessToken } from "@/utils/auth";
import { buildMeetingPreviewUrl } from "@/utils/meeting"; import { buildMeetingPreviewUrl } from "@/utils/meeting";
const { Paragraph, Title } = Typography; const { Paragraph, Title } = Typography;
export default function MeetingPreviewPage() { export default function MeetingPreviewPage() {
const { message } = App.useApp(); const { message } = App.useApp();
const navigate = useNavigate();
const { id } = useParams<{ id: string }>(); const { id } = useParams<{ id: string }>();
const [searchParams] = useSearchParams(); const [searchParams] = useSearchParams();
const [loading, setLoading] = useState(true); const [loading, setLoading] = useState(true);
@ -27,6 +30,10 @@ export default function MeetingPreviewPage() {
const presetAccessPassword = useMemo(() => (searchParams.get("accessPassword") || "").trim(), [searchParams]); const presetAccessPassword = useMemo(() => (searchParams.get("accessPassword") || "").trim(), [searchParams]);
usePageTitle(meeting?.title || "会议预览"); usePageTitle(meeting?.title || "会议预览");
const handleBack = () => {
navigate(hasAccessToken() ? "/meetings" : "/login", { replace: true });
};
const loadPreview = async (password?: string) => { const loadPreview = async (password?: string) => {
const previewResp = await getPublicMeetingPreview(meetingId, password); const previewResp = await getPublicMeetingPreview(meetingId, password);
setMeeting(previewResp.data.data.meeting); setMeeting(previewResp.data.data.meeting);
@ -69,7 +76,7 @@ export default function MeetingPreviewPage() {
}; };
void run(); void run();
}, [meetingId, presetAccessPassword]); }, [meetingId, presetAccessPassword, message]);
const handleSubmitPassword = async () => { const handleSubmitPassword = async () => {
if (!accessPassword.trim()) { if (!accessPassword.trim()) {
@ -93,21 +100,24 @@ export default function MeetingPreviewPage() {
if (passwordRequired && !passwordVerified) { if (passwordRequired && !passwordVerified) {
return ( return (
<div className="preview-page"> <div className="preview-page">
<Card className="surface-card preview-password-card"> <div className="preview-page__inner">
<Title level={3}></Title> <PageHeader title="分享预览" back onBack={handleBack} />
<Paragraph type="secondary">访访</Paragraph> <Card className="surface-card preview-password-card">
<Space direction="vertical" size={16} style={{ width: "100%" }}> <Title level={3}></Title>
<Input.Password <Paragraph type="secondary">访访</Paragraph>
value={accessPassword} <Space direction="vertical" size={16} style={{ width: "100%" }}>
placeholder="请输入访问密码" <Input.Password
onChange={(event) => setAccessPassword(event.target.value)} value={accessPassword}
onPressEnter={() => void handleSubmitPassword()} placeholder="请输入访问密码"
/> onChange={(event) => setAccessPassword(event.target.value)}
<Button type="primary" block loading={loading} onClick={() => void handleSubmitPassword()}> onPressEnter={() => void handleSubmitPassword()}
/>
</Button> <Button type="primary" block loading={loading} onClick={() => void handleSubmitPassword()}>
</Space>
</Card> </Button>
</Space>
</Card>
</div>
</div> </div>
); );
} }
@ -119,9 +129,7 @@ export default function MeetingPreviewPage() {
return ( return (
<div className="preview-page"> <div className="preview-page">
<div className="preview-page__inner"> <div className="preview-page__inner">
<div className="preview-page__header"> <PageHeader title="分享预览" back onBack={handleBack} />
<span className="login-page__badge">iMeeting </span>
</div>
<MeetingPreviewView <MeetingPreviewView
meeting={meeting} meeting={meeting}
transcripts={transcripts} transcripts={transcripts}

View File

@ -47,6 +47,7 @@ export interface MeetingVO {
summaryDetailLevel?: "DETAILED" | "STANDARD" | "BRIEF"; summaryDetailLevel?: "DETAILED" | "STANDARD" | "BRIEF";
summaryModelId?: number; summaryModelId?: number;
promptId?: number; promptId?: number;
aiCatalogEnabled?: boolean;
audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED"; audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED";
audioSaveMessage?: string; audioSaveMessage?: string;
accessPassword?: string | null; accessPassword?: string | null;