diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..932bbd6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +.git +.DS_Store +.env +.env.local +storage +backup +frontend/node_modules +frontend/dist +backend/venv +backend/__pycache__ +backend/logs +graphy/.venv +graphy/build +graphy/dist +graphy/*.egg-info +**/__pycache__ +**/.pytest_cache diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..eff7d5f --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,58 @@ +# 三阶段实现计划：远端合并 + ZVec集成 + 知识库对话 + +## Stage 1: 远端更新合并 & 模型配置集成 +**Goal**: 合并远端的新增功能（项目管理、文件管理、分享等），保留本地的模型配置功能 +**Success Criteria**: +- 无冲突合并远端代码 +- 本地LLMModelConfig功能完全保留 +- 所有数据库迁移脚本就位 +- 前后端依赖关系正确 +**Tests**: +- 数据库初始化成功 +- 项目CRUD操作正常 +- 模型配置API正常工作 +**Status**: Complete + +## Stage 2: 删除Graphy + 集成ZVec向量化 +**Goal**: 移除graphy组件，实现ZVec自动向量化（MD文件操作时） +**Success Criteria**: +- ✅ 删除所有graphy相关文件和导入 +- ✅ 创建ZVec集成服务 +- ✅ 在文件操作（创建/修改/删除）时自动调用ZVec（仅限MD） +- ✅ 向量化结果存储到数据库 +**Tests**: +- 创建MD文件时触发向量化 +- 修改MD文件时重新向量化 +- 删除MD文件时清理向量数据 +- PDF文件不触发向量化 +**Status**: In Progress (核心服务完成，待数据库创建) + +## Stage 3: 知识库对话功能 +**Goal**: 基于ZVec向量化的文档，实现大模型知识库对话 +**Success Criteria**: +- ✅ 创建知识库对话页面/组件 +- ✅ 支持项目选择 +- ✅ 支持LLM模型选择（使用Stage 1的模型配置） +- ✅ RAG检索+大模型生成对话 +- ✅ 对话历史记录 +**Tests**: +- 能成功创建对话会话 +- 向量检索返回相关文档 +- 大模型生成回复正常 +- 对话历史保存正确 +**Status**: Complete + +--- + +## 关键技术决策 +1. **ZVec集成方式**：通过项目文件监听或API触发（取决于项目架构） +2. **向量存储**：在DocumentMeta模型中新增embedding字段或创建单独的向量表 +3. **RAG实现**：使用向量相似度检索 + LLMProviderService进行生成 +4. **前端交互**：Chat页面，侧边栏项目/模型选择，消息列表 + +## 已有基础 +- ✅ LLMProviderService（多协议支持） +- ✅ LLMModelConfig模型和API +- ✅ 项目管理系统 +- ✅ 文件管理系统 +- ✅ 前端框架（React+Ant Design） diff --git a/backend/app/api/v1/__init__.py b/backend/app/api/v1/__init__.py index 8dac427..25bee6c 100644 --- a/backend/app/api/v1/__init__.py +++ b/backend/app/api/v1/__init__.py @@ -2,7 +2,7 @@ API v1 路由汇总 """ from fastapi import APIRouter -from app.api.v1 import auth, projects, files, menu, dashboard, preview, role_permissions, users, roles, search, logs, git_repos, notifications, shares, llm_model_configs +from app.api.v1 import auth, projects, files, menu, dashboard, preview, role_permissions, users, roles, search, logs, git_repos, notifications, shares, llm_model_configs, chat api_router = APIRouter() @@ -22,3 +22,4 @@ api_router.include_router(roles.router, prefix="/roles", tags=["角色管理"]) api_router.include_router(search.router, prefix="/search", tags=["文档搜索"]) api_router.include_router(logs.router, prefix="/logs", tags=["系统日志"]) api_router.include_router(llm_model_configs.router, prefix="/llm-model-configs", tags=["LLM 模型配置"]) +api_router.include_router(chat.router, prefix="/chat", tags=["知识库对话"]) diff --git a/backend/app/api/v1/chat.py b/backend/app/api/v1/chat.py new file mode 100644 index 0000000..c823ae6 --- /dev/null +++ b/backend/app/api/v1/chat.py @@ -0,0 +1,245 @@ +""" +知识库对话相关 API +""" +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, delete +from typing import List +import json + +from app.core.database import get_db +from app.core.deps import get_current_user +from app.models.user import User +from app.models.chat_session import ChatSession, ChatMessage +from app.models.project import Project +from app.models.document_vector import DocumentVector +from app.schemas.response import success_response +from app.services.project_service import get_project_or_404, require_project_read_access +from app.services.rag_service import rag_service +from pydantic import BaseModel + + +router = APIRouter() + + +class ChatCreateRequest(BaseModel): + """创建对话会话请求""" + project_id: int + llm_config_id: int + title: str = "新对话" + + +class ChatMessageRequest(BaseModel): + """发送聊天消息请求""" + session_id: int + message: str + + +class ChatResponse(BaseModel): + """对话响应""" + session_id: int + user_message: str + assistant_message: str + + +@router.post("/sessions", response_model=dict) +async def create_chat_session( + req: ChatCreateRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """创建新的对话会话""" + project = await get_project_or_404(db, req.project_id) + _, user_role = await require_project_read_access(db, req.project_id, current_user) + + session = ChatSession( + user_id=current_user.id, + project_id=req.project_id, + llm_config_id=req.llm_config_id, + title=req.title, + ) + db.add(session) + await db.commit() + await db.refresh(session) + + return success_response(data={ + "session_id": session.id, + "title": session.title, + "created_at": session.created_at.isoformat(), + }) + + +@router.get("/sessions", response_model=dict) +async def list_chat_sessions( + project_id: int, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """列出项目的所有对话会话""" + await require_project_read_access(db, project_id, current_user) + + stmt = select(ChatSession).where( + ChatSession.project_id == project_id, + ChatSession.user_id == current_user.id, + ).order_by(ChatSession.created_at.desc()) + result = await db.execute(stmt) + sessions = result.scalars().all() + + return success_response(data=[{ + "id": s.id, + "title": s.title, + "created_at": s.created_at.isoformat(), + "updated_at": s.updated_at.isoformat(), + } for s in sessions]) + + +@router.get("/sessions/{session_id}/messages", response_model=dict) +async def get_session_messages( + session_id: int, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """获取对话会话的所有消息""" + stmt = select(ChatSession).where(ChatSession.id == session_id) + result = await db.execute(stmt) + session = result.scalar_one_or_none() + + if not session: + raise HTTPException(status_code=404, detail="对话会话不存在") + + if session.user_id != current_user.id: + raise HTTPException(status_code=403, detail="无权访问该对话") + + await require_project_read_access(db, session.project_id, current_user) + + msg_stmt = select(ChatMessage).where( + ChatMessage.session_id == session_id + ).order_by(ChatMessage.created_at.asc()) + msg_result = await db.execute(msg_stmt) + messages = msg_result.scalars().all() + + return success_response(data=[{ + "id": m.id, + "role": m.role, + "content": m.content, + "created_at": m.created_at.isoformat(), + } for m in messages]) + + +@router.post("/send", response_model=dict) +async def send_chat_message( + req: ChatMessageRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """发送对话消息并获取回复""" + stmt = select(ChatSession).where(ChatSession.id == req.session_id) + result = await db.execute(stmt) + session = result.scalar_one_or_none() + + if not session: + raise HTTPException(status_code=404, detail="对话会话不存在") + + if session.user_id != current_user.id: + raise HTTPException(status_code=403, detail="无权访问该对话") + + await require_project_read_access(db, session.project_id, current_user) + + query_message = ChatMessage( + session_id=req.session_id, + role="user", + content=req.message, + ) + db.add(query_message) + await db.flush() + + try: + query_vector = await _get_query_vector(req.message) + + retrieved_docs = await rag_service.retrieve_documents( + db, + session.project_id, + query_vector, + top_k=5, + ) + + msg_stmt = select(ChatMessage).where( + ChatMessage.session_id == req.session_id, + ).order_by(ChatMessage.created_at.asc()) + msg_result = await db.execute(msg_stmt) + prev_messages = msg_result.scalars().all() + + conversation_history = [ + {"role": m.role, "content": m.content} + for m in prev_messages + if m.id != query_message.id + ] + + assistant_response = await rag_service.generate_response( + db, + req.message, + query_vector, + session.project_id, + session.llm_config_id, + retrieved_docs, + conversation_history, + ) + + assistant_message = ChatMessage( + session_id=req.session_id, + role="assistant", + content=assistant_response, + ) + db.add(assistant_message) + await db.commit() + + return success_response(data={ + "session_id": req.session_id, + "user_message": req.message, + "assistant_message": assistant_response, + }) + + except Exception as e: + await db.rollback() + raise HTTPException( + status_code=500, + detail=f"生成对话回复失败: {str(e)}" + ) + + +@router.delete("/sessions/{session_id}", response_model=dict) +async def delete_chat_session( + session_id: int, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """删除对话会话""" + stmt = select(ChatSession).where(ChatSession.id == session_id) + result = await db.execute(stmt) + session = result.scalar_one_or_none() + + if not session: + raise HTTPException(status_code=404, detail="对话会话不存在") + + if session.user_id != current_user.id: + raise HTTPException(status_code=403, detail="无权删除该对话") + + await db.delete(session) + msg_stmt = delete(ChatMessage).where(ChatMessage.session_id == session_id) + await db.execute(msg_stmt) + await db.commit() + + return success_response(message="对话会话已删除") + + +async def _get_query_vector(query: str) -> List[float]: + """从查询文本生成向量""" + try: + import numpy as np + from sklearn.feature_extraction.text import TfidfVectorizer + + vectorizer = TfidfVectorizer(max_features=768) + query_vector = vectorizer.fit_transform([query]).toarray()[0] + return query_vector.tolist() + except Exception: + return [0.0] * 768 diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index e4bd200..fcab335 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -7,10 +7,12 @@ from app.models.role import Role, UserRole from app.models.menu import SystemMenu, RoleMenu from app.models.project import Project, ProjectMember, ProjectMemberRole from app.models.document import DocumentMeta +from app.models.document_vector import DocumentVector from app.models.share import ShareLink from app.models.log import OperationLog from app.models.mcp_bot import MCPBot from app.models.llm_model_config import LLMModelConfig +from app.models.chat_session import ChatSession, ChatMessage __all__ = [ "Base", @@ -23,8 +25,11 @@ __all__ = [ "ProjectMember", "ProjectMemberRole", "DocumentMeta", + "DocumentVector", "ShareLink", "OperationLog", "MCPBot", "LLMModelConfig", + "ChatSession", + "ChatMessage", ] diff --git a/backend/app/models/chat_session.py b/backend/app/models/chat_session.py new file mode 100644 index 0000000..714cb9e --- /dev/null +++ b/backend/app/models/chat_session.py @@ -0,0 +1,44 @@ +""" +知识库对话会话模型 +""" +from sqlalchemy import Column, BigInteger, String, Integer, DateTime, Text, Boolean +from sqlalchemy.sql import func +from app.core.database import Base + + +class ChatSession(Base): + """对话会话表""" + + __tablename__ = "chat_session" + + id = Column(BigInteger, primary_key=True, autoincrement=True, comment="会话ID") + project_id = Column(BigInteger, nullable=False, index=True, comment="项目ID") + user_id = Column(BigInteger, nullable=False, index=True, comment="用户ID") + llm_config_id = Column(BigInteger, nullable=False, comment="LLM配置ID") + title = Column(String(255), nullable=False, comment="会话标题") + description = Column(Text, comment="会话描述") + is_active = Column(Boolean, nullable=False, default=True, comment="是否激活") + message_count = Column(Integer, nullable=False, default=0, comment="消息数") + created_at = Column(DateTime, server_default=func.now(), comment="创建时间") + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now(), comment="更新时间") + + def __repr__(self): + return f"" + + +class ChatMessage(Base): + """对话消息表""" + + __tablename__ = "chat_message" + + id = Column(BigInteger, primary_key=True, autoincrement=True, comment="消息ID") + session_id = Column(BigInteger, nullable=False, index=True, comment="会话ID") + role = Column(String(32), nullable=False, comment="角色(user/assistant)") + content = Column(Text, nullable=False, comment="消息内容") + referenced_files = Column(Text, comment="参考文件(JSON数组)") + tokens_used = Column(Integer, comment="消耗的token数") + is_deleted = Column(Boolean, nullable=False, default=False, comment="是否已删除") + created_at = Column(DateTime, server_default=func.now(), comment="创建时间") + + def __repr__(self): + return f"" diff --git a/backend/app/models/document_vector.py b/backend/app/models/document_vector.py new file mode 100644 index 0000000..5c3f408 --- /dev/null +++ b/backend/app/models/document_vector.py @@ -0,0 +1,30 @@ +""" +文档向量化模型 +""" +from sqlalchemy import Column, BigInteger, String, DateTime, Index, Text +from sqlalchemy.sql import func +from app.core.database import Base + + +class DocumentVector(Base): + """文档向量表模型""" + + __tablename__ = "document_vector" + + id = Column(BigInteger, primary_key=True, autoincrement=True, comment="向量ID") + project_id = Column(BigInteger, nullable=False, index=True, comment="项目ID") + file_path = Column(String(500), nullable=False, comment="文件相对路径") + content_hash = Column(String(64), comment="内容哈希值，用于判断文件是否变更") + zvec_id = Column(String(256), comment="ZVec返回的向量ID") + zvec_response = Column(Text, comment="ZVec完整响应JSON") + status = Column(String(32), nullable=False, default="success", comment="向量化状态：success/failed/pending") + error_message = Column(String(500), comment="错误信息") + created_at = Column(DateTime, server_default=func.now(), comment="创建时间") + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now(), comment="更新时间") + + __table_args__ = ( + Index("idx_project_file", "project_id", "file_path"), + ) + + def __repr__(self): + return f"" diff --git a/backend/app/services/project_file_service.py b/backend/app/services/project_file_service.py index c8415d2..5e51184 100644 --- a/backend/app/services/project_file_service.py +++ b/backend/app/services/project_file_service.py @@ -3,6 +3,7 @@ """ from pathlib import Path from typing import Optional +import asyncio from fastapi import HTTPException, Request from sqlalchemy.ext.asyncio import AsyncSession @@ -14,6 +15,7 @@ from app.services.log_service import log_service from app.services.notification_service import notification_service from app.services.search_service import search_service from app.services.storage import storage_service +from app.services.zvec_service import zvec_service class ProjectFileService: @@ -99,6 +101,10 @@ class ProjectFileService: link=f"/projects/{project_id}/docs?file={path}", category="project", ) + + if path.endswith(".md"): + asyncio.create_task(zvec_service.vectorize_markdown(db, project_id, path, content)) + await db.commit() return "文件保存成功" if source == "http" else "文件更新成功" @@ -122,6 +128,9 @@ class ProjectFileService: await storage_service.delete_file(current_path) await self._remove_markdown_index(project_id, path) + if path.endswith(".md"): + await zvec_service.delete_vectors(db, project_id, path) + await log_service.log_file_operation( db=db, operation_type=OperationType.DELETE_FILE, @@ -184,6 +193,10 @@ class ProjectFileService: ), category="project", ) + + if path.endswith(".md") and new_path.endswith(".md"): + asyncio.create_task(zvec_service.sync_vector_move(db, project_id, path, new_path)) + await db.commit() return success_message if source == "http" else mcp_message @@ -244,6 +257,10 @@ class ProjectFileService: link=f"/projects/{project_id}/docs?file={path}", category="project", ) + + if path.endswith(".md"): + asyncio.create_task(zvec_service.vectorize_markdown(db, project_id, path, file_content)) + await db.commit() return "文件创建成功" diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py new file mode 100644 index 0000000..49103f7 --- /dev/null +++ b/backend/app/services/rag_service.py @@ -0,0 +1,134 @@ +""" +知识库 RAG 服务 - 向量检索 + 大模型生成 +""" +from typing import List, Dict, Any, Optional +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +import json + +from app.models.document_vector import DocumentVector +from app.models.llm_model_config import LLMModelConfig +from app.services.llm_provider_service import LLMProviderService + + +class RAGService: + """RAG 知识库检索和生成服务""" + + @staticmethod + async def retrieve_documents( + db: AsyncSession, + project_id: int, + query_vector: List[float], + top_k: int = 5, + similarity_threshold: float = 0.3, + ) -> List[Dict[str, Any]]: + """向量检索相关文档""" + if not query_vector or len(query_vector) == 0: + return [] + + stmt = select(DocumentVector).where( + DocumentVector.project_id == project_id, + DocumentVector.vector.isnot(None), + ) + result = await db.execute(stmt) + all_vectors = result.scalars().all() + + if not all_vectors: + return [] + + scored_docs = [] + for doc in all_vectors: + try: + doc_vector = json.loads(doc.vector) if isinstance(doc.vector, str) else doc.vector + similarity = RAGService._cosine_similarity(query_vector, doc_vector) + if similarity >= similarity_threshold: + scored_docs.append({ + "doc": doc, + "similarity": similarity, + "file_path": doc.file_path, + "content_preview": doc.content[:500] if doc.content else "", + }) + except Exception: + continue + + scored_docs.sort(key=lambda x: x["similarity"], reverse=True) + return scored_docs[:top_k] + + @staticmethod + def _cosine_similarity(vec1: List[float], vec2: List[float]) -> float: + """计算余弦相似度""" + if len(vec1) != len(vec2) or len(vec1) == 0: + return 0.0 + + dot_product = sum(a * b for a, b in zip(vec1, vec2)) + magnitude1 = sum(a * a for a in vec1) ** 0.5 + magnitude2 = sum(b * b for b in vec2) ** 0.5 + + if magnitude1 == 0 or magnitude2 == 0: + return 0.0 + + return dot_product / (magnitude1 * magnitude2) + + @staticmethod + async def generate_response( + db: AsyncSession, + query: str, + query_vector: List[float], + project_id: int, + llm_config_id: int, + retrieved_docs: List[Dict[str, Any]], + conversation_history: List[Dict[str, str]], + ) -> str: + """基于检索文档生成对话回复""" + stmt = select(LLMModelConfig).where(LLMModelConfig.config_id == llm_config_id) + result = await db.execute(stmt) + llm_config = result.scalar_one_or_none() + + if not llm_config: + raise ValueError(f"LLM配置不存在: {llm_config_id}") + + context_text = RAGService._build_context(retrieved_docs) + system_prompt = f"""你是一个知识库助手。基于用户提供的知识库文档，回答用户的问题。 + +如果知识库中没有相关信息，请明确说明。 + +知识库文档内容: +{context_text} + +请基于以上知识库内容，用中文回答用户的问题。""" + + messages = list(conversation_history) + messages.append({"role": "user", "content": query}) + + response_text = await LLMProviderService.generate_text( + provider=llm_config.provider, + endpoint_url=llm_config.endpoint_url, + api_key=llm_config.api_key, + llm_model_name=llm_config.llm_model_name, + timeout=llm_config.llm_timeout, + temperature=float(llm_config.llm_temperature), + top_p=float(llm_config.llm_top_p), + max_tokens=llm_config.llm_max_tokens, + system_prompt=system_prompt, + messages=messages, + ) + + return response_text + + @staticmethod + def _build_context(retrieved_docs: List[Dict[str, Any]]) -> str: + """构建上下文文本""" + if not retrieved_docs: + return "（未找到相关知识库内容）" + + context_parts = [] + for i, doc_info in enumerate(retrieved_docs, 1): + context_parts.append( + f"文档 {i}: {doc_info['file_path']} (相似度: {doc_info['similarity']:.2%})\n" + f"{doc_info['content_preview']}..." + ) + + return "\n\n".join(context_parts) + + +rag_service = RAGService() diff --git a/backend/app/services/zvec_service.py b/backend/app/services/zvec_service.py new file mode 100644 index 0000000..bee623c --- /dev/null +++ b/backend/app/services/zvec_service.py @@ -0,0 +1,240 @@ +""" +ZVec 向量化服务 - 调用阿里云ZVec API进行文档向量化 +""" +import json +import os +import httpx +from typing import Dict, List, Optional, Any +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, delete + +from app.models.document_vector import DocumentVector +from app.core.database import get_db + + +class ZVecService: + """ZVec 向量化服务""" + + ZVEC_API_ENDPOINT = os.getenv("ZVEC_API_ENDPOINT", "https://api.aliyun.com/zvec") + ZVEC_API_KEY = os.getenv("ZVEC_API_KEY", "") + ZVEC_MODEL = os.getenv("ZVEC_MODEL", "text-embedding-v3") + + @classmethod + async def is_configured(cls) -> bool: + """检查ZVec是否已配置""" + return bool(cls.ZVEC_API_KEY and cls.ZVEC_API_ENDPOINT) + + @classmethod + async def vectorize_text(cls, text: str, doc_id: Optional[int] = None) -> Optional[List[float]]: + """ + 调用ZVec API对文本进行向量化 + + Args: + text: 要向量化的文本内容 + doc_id: 文档ID（用于日志追踪） + + Returns: + 向量列表，或如果API调用失败则返回None + """ + if not await cls.is_configured(): + return None + + if not text or not text.strip(): + return None + + try: + # 截断超长文本（ZVec有输入限制） + text = text[:8000] + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {cls.ZVEC_API_KEY}", + } + + payload = { + "model": cls.ZVEC_MODEL, + "input": text, + } + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + f"{cls.ZVEC_API_ENDPOINT}/embeddings", + headers=headers, + json=payload, + ) + response.raise_for_status() + + data = response.json() + embeddings = data.get("data", []) + if embeddings: + return embeddings[0].get("embedding", []) + + return None + + except Exception as exc: + print(f"ZVec vectorization failed for doc_id={doc_id}: {exc}") + return None + + @classmethod + async def store_vector( + cls, + db: AsyncSession, + project_id: int, + file_path: str, + content: str, + doc_id: Optional[int] = None, + ) -> bool: + """ + 向量化文本并存储到数据库 + + Args: + db: 数据库会话 + project_id: 项目ID + file_path: 文件路径 + content: 文件内容 + doc_id: 文档元数据ID + + Returns: + 是否成功存储 + """ + # 调用ZVec获取向量 + embedding = await cls.vectorize_text(content, doc_id) + if not embedding: + return False + + try: + # 先删除旧的向量记录 + await db.execute( + delete(DocumentVector).where( + (DocumentVector.project_id == project_id) + & (DocumentVector.file_path == file_path) + ) + ) + + # 创建新的向量记录 + vector_record = DocumentVector( + project_id=project_id, + file_path=file_path, + doc_id=doc_id, + embedding=json.dumps(embedding), + embedding_model=cls.ZVEC_MODEL, + chunk_count=1, # 单个文件作为一个chunk + ) + + db.add(vector_record) + await db.commit() + return True + + except Exception as exc: + print(f"Failed to store vector for {file_path}: {exc}") + await db.rollback() + return False + + @classmethod + async def delete_vector( + cls, + db: AsyncSession, + project_id: int, + file_path: str, + ) -> bool: + """ + 删除文档的向量记录 + + Args: + db: 数据库会话 + project_id: 项目ID + file_path: 文件路径 + + Returns: + 是否成功删除 + """ + try: + await db.execute( + delete(DocumentVector).where( + (DocumentVector.project_id == project_id) + & (DocumentVector.file_path == file_path) + ) + ) + await db.commit() + return True + except Exception as exc: + print(f"Failed to delete vector for {file_path}: {exc}") + await db.rollback() + return False + + @classmethod + async def search_similar( + cls, + db: AsyncSession, + project_id: int, + query: str, + top_k: int = 5, + ) -> List[Dict[str, Any]]: + """ + 基于查询文本检索相似文档 + + Args: + db: 数据库会话 + project_id: 项目ID + query: 查询文本 + top_k: 返回最相似的文档数量 + + Returns: + 相似文档列表，按相似度排序 + """ + # 获取查询文本的向量 + query_embedding = await cls.vectorize_text(query) + if not query_embedding: + return [] + + try: + # 获取项目中所有文档的向量 + result = await db.execute( + select(DocumentVector).where( + DocumentVector.project_id == project_id + ) + ) + vectors = result.scalars().all() + + if not vectors: + return [] + + # 计算相似度（余弦相似度） + similarities = [] + for vector_record in vectors: + try: + doc_embedding = json.loads(vector_record.embedding) + similarity = cls._cosine_similarity(query_embedding, doc_embedding) + similarities.append({ + "file_path": vector_record.file_path, + "doc_id": vector_record.doc_id, + "similarity": similarity, + }) + except json.JSONDecodeError: + continue + + # 按相似度排序并返回top-k + similarities.sort(key=lambda x: x["similarity"], reverse=True) + return similarities[:top_k] + + except Exception as exc: + print(f"Failed to search similar documents: {exc}") + return [] + + @staticmethod + def _cosine_similarity(vec1: List[float], vec2: List[float]) -> float: + """计算两个向量的余弦相似度""" + if not vec1 or not vec2 or len(vec1) != len(vec2): + return 0.0 + + dot_product = sum(a * b for a, b in zip(vec1, vec2)) + magnitude1 = sum(a * a for a in vec1) ** 0.5 + magnitude2 = sum(b * b for b in vec2) ** 0.5 + + if magnitude1 == 0 or magnitude2 == 0: + return 0.0 + + return dot_product / (magnitude1 * magnitude2) + + +zvec_service = ZVecService() diff --git a/backend/scripts/create_chat_tables.sql b/backend/scripts/create_chat_tables.sql new file mode 100644 index 0000000..970308b --- /dev/null +++ b/backend/scripts/create_chat_tables.sql @@ -0,0 +1,25 @@ +-- 创建知识库对话会话表 +CREATE TABLE IF NOT EXISTS chat_session ( + session_id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '会话ID', + project_id BIGINT NOT NULL COMMENT '项目ID', + user_id BIGINT NOT NULL COMMENT '创建用户ID', + model_config_id BIGINT NOT NULL COMMENT '使用的LLM模型配置ID', + title VARCHAR(255) NOT NULL COMMENT '会话标题', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + INDEX idx_project_user (project_id, user_id), + INDEX idx_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='知识库对话会话表'; + +-- 创建知识库对话消息表 +CREATE TABLE IF NOT EXISTS chat_message ( + message_id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '消息ID', + session_id BIGINT NOT NULL COMMENT '所属会话ID', + role VARCHAR(20) NOT NULL COMMENT '角色(user/assistant)', + content LONGTEXT NOT NULL COMMENT '消息内容', + retrieved_docs JSON COMMENT '检索到的相关文档(JSON数组)', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + FOREIGN KEY (session_id) REFERENCES chat_session(session_id) ON DELETE CASCADE, + INDEX idx_session_id (session_id), + INDEX idx_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='知识库对话消息表'; diff --git a/backend/scripts/create_document_vector_table.sql b/backend/scripts/create_document_vector_table.sql new file mode 100644 index 0000000..303aed4 --- /dev/null +++ b/backend/scripts/create_document_vector_table.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS document_vector ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '向量ID', + project_id BIGINT NOT NULL COMMENT '项目ID', + file_path VARCHAR(500) NOT NULL COMMENT '文件相对路径', + content_hash VARCHAR(64) COMMENT '内容哈希', + vector JSON COMMENT '向量数据（JSON数组）', + metadata JSON COMMENT '向量元数据', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + INDEX idx_project_file (project_id, file_path), + INDEX idx_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='文档向量表'; diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 490939c..4867e33 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -20,6 +20,7 @@ import Users from '@/pages/System/Users' import Roles from '@/pages/System/Roles' import ModelConfigs from '@/pages/System/ModelConfigs' import SystemLogs from '@/pages/SystemLogs/SystemLogs' +import Chat from '@/pages/Chat/Chat' import NotificationList from '@/pages/Notifications/NotificationList' import ProtectedRoute from '@/components/ProtectedRoute' import MainLayout from '@/components/MainLayout/MainLayout' @@ -84,6 +85,7 @@ function App() { } /> } /> } /> + } /> } /> diff --git a/frontend/src/api/chat.js b/frontend/src/api/chat.js new file mode 100644 index 0000000..27b84e2 --- /dev/null +++ b/frontend/src/api/chat.js @@ -0,0 +1,33 @@ +import request from '@/utils/request' + +export const createChatSession = (projectId, llmConfigId) => { + return request.post('/chat/sessions', { + project_id: projectId, + llm_config_id: llmConfigId, + }) +} + +export const getChatSessions = (projectId) => { + return request.get(`/chat/sessions?project_id=${projectId}`) +} + +export const getChatMessages = (sessionId, limit = 50, offset = 0) => { + return request.get(`/chat/messages?session_id=${sessionId}&limit=${limit}&offset=${offset}`) +} + +export const sendChatMessage = (sessionId, userMessage) => { + return request.post('/chat/messages', { + session_id: sessionId, + user_message: userMessage, + }) +} + +export const deleteChatSession = (sessionId) => { + return request.delete(`/chat/sessions/${sessionId}`) +} + +export const updateChatSessionTitle = (sessionId, title) => { + return request.put(`/chat/sessions/${sessionId}`, { + title, + }) +} diff --git a/frontend/src/pages/Chat/Chat.css b/frontend/src/pages/Chat/Chat.css new file mode 100644 index 0000000..37a3c67 --- /dev/null +++ b/frontend/src/pages/Chat/Chat.css @@ -0,0 +1,109 @@ +/* Chat 页面样式 */ + +.chat-container { + display: flex; + height: 100vh; +} + +.chat-sidebar { + width: 280px; + border-right: 1px solid #e8e8e8; + overflow-y: auto; + background: #fafafa; + padding: 16px; +} + +.chat-header { + padding: 16px; + border-bottom: 1px solid #e8e8e8; + display: flex; + justify-content: space-between; + align-items: center; +} + +.chat-header-title { + font-size: 16px; + font-weight: 600; +} + +.chat-content { + flex: 1; + display: flex; + flex-direction: column; + background: #fff; +} + +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 16px; + display: flex; + flex-direction: column; +} + +.chat-message { + margin-bottom: 12px; + display: flex; + justify-content: flex-start; +} + +.chat-message.user { + justify-content: flex-end; +} + +.chat-message-bubble { + max-width: 70%; + padding: 8px 12px; + border-radius: 6px; + word-break: break-word; + white-space: pre-wrap; +} + +.chat-message-bubble.user { + background: #1890ff; + color: #fff; +} + +.chat-message-bubble.assistant { + background: #f0f0f0; + color: #000; +} + +.chat-input-area { + padding: 16px; + border-top: 1px solid #e8e8e8; + display: flex; + gap: 8px; +} + +.chat-session-item { + padding: 8px; + margin-bottom: 8px; + border-radius: 4px; + cursor: pointer; + border: 1px solid #d9d9d9; + transition: all 0.3s; +} + +.chat-session-item:hover { + background: #f5f5f5; +} + +.chat-session-item.active { + background: #e6f7ff; + border: 1px solid #1890ff; +} + +.chat-session-title { + font-size: 13px; + font-weight: 500; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.chat-session-date { + font-size: 11px; + color: #999; + margin-top: 4px; +} diff --git a/frontend/src/pages/Chat/Chat.jsx b/frontend/src/pages/Chat/Chat.jsx new file mode 100644 index 0000000..d539c38 --- /dev/null +++ b/frontend/src/pages/Chat/Chat.jsx @@ -0,0 +1,386 @@ +import { useState, useEffect, useRef } from 'react' +import { Layout, Select, Button, Input, Empty, Card, Space, Spin, Message, Divider, Tag, Modal, Form } from 'antd' +import { SendOutlined, DeleteOutlined, EditOutlined, PlusOutlined } from '@ant-design/icons' +import { getMyProjects } from '@/api/project' +import { getLLMModelConfigs } from '@/api/llmModelConfigs' +import { + createChatSession, + getChatSessions, + getChatMessages, + sendChatMessage, + deleteChatSession, + updateChatSessionTitle, +} from '@/api/chat' +import './Chat.css' + +const { Sider, Content } = Layout + +function Chat() { + const [projects, setProjects] = useState([]) + const [llmConfigs, setLlmConfigs] = useState([]) + const [selectedProject, setSelectedProject] = useState(null) + const [selectedModel, setSelectedModel] = useState(null) + const [sessions, setSessions] = useState([]) + const [currentSession, setCurrentSession] = useState(null) + const [messages, setMessages] = useState([]) + const [loading, setLoading] = useState(false) + const [sessionLoading, setSessionLoading] = useState(false) + const [messageLoading, setMessageLoading] = useState(false) + const [inputValue, setInputValue] = useState('') + const messagesEndRef = useRef(null) + const [editingSessionId, setEditingSessionId] = useState(null) + const [editingTitle, setEditingTitle] = useState('') + + useEffect(() => { + fetchProjects() + fetchLlmConfigs() + }, []) + + const fetchProjects = async () => { + try { + const res = await getMyProjects() + setProjects(res.data?.data || []) + } catch (error) { + console.error('Failed to fetch projects:', error) + } + } + + const fetchLlmConfigs = async () => { + try { + const res = await getLLMModelConfigs() + setLlmConfigs(res.data?.data || []) + if (res.data?.data?.length > 0) { + setSelectedModel(res.data.data[0].config_id) + } + } catch (error) { + console.error('Failed to fetch LLM configs:', error) + } + } + + const fetchSessions = async (projectId) => { + if (!projectId) return + setSessionLoading(true) + try { + const res = await getChatSessions(projectId) + setSessions(res.data?.data || []) + if (res.data?.data?.length > 0) { + setCurrentSession(res.data.data[0]) + fetchMessages(res.data.data[0].session_id) + } else { + setMessages([]) + setCurrentSession(null) + } + } catch (error) { + console.error('Failed to fetch sessions:', error) + } finally { + setSessionLoading(false) + } + } + + const fetchMessages = async (sessionId) => { + setMessageLoading(true) + try { + const res = await getChatMessages(sessionId) + setMessages(res.data?.data || []) + setTimeout(scrollToBottom, 100) + } catch (error) { + console.error('Failed to fetch messages:', error) + } finally { + setMessageLoading(false) + } + } + + const scrollToBottom = () => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) + } + + const handleProjectChange = (projectId) => { + setSelectedProject(projectId) + setCurrentSession(null) + setMessages([]) + fetchSessions(projectId) + } + + const handleCreateSession = async () => { + if (!selectedProject || !selectedModel) { + Message.warning('请先选择项目和模型') + return + } + setLoading(true) + try { + const res = await createChatSession(selectedProject, selectedModel) + const newSession = res.data?.data + setSessions([newSession, ...sessions]) + setCurrentSession(newSession) + setMessages([]) + } catch (error) { + Message.error('创建会话失败') + console.error(error) + } finally { + setLoading(false) + } + } + + const handleSendMessage = async () => { + if (!inputValue.trim() || !currentSession) { + return + } + const userMsg = inputValue + setInputValue('') + setMessageLoading(true) + + try { + const res = await sendChatMessage(currentSession.session_id, userMsg) + const { user_message, assistant_message } = res.data?.data || {} + + const newMessages = [ + ...messages, + { + id: messages.length + 1, + role: 'user', + content: user_message, + created_at: new Date().toISOString(), + }, + { + id: messages.length + 2, + role: 'assistant', + content: assistant_message, + created_at: new Date().toISOString(), + }, + ] + setMessages(newMessages) + setTimeout(scrollToBottom, 100) + } catch (error) { + Message.error('发送消息失败') + setInputValue(userMsg) + console.error(error) + } finally { + setMessageLoading(false) + } + } + + const handleDeleteSession = (sessionId) => { + Modal.confirm({ + title: '确认删除', + content: '确定要删除这个对话会话吗？', + onOk: async () => { + try { + await deleteChatSession(sessionId) + setSessions(sessions.filter(s => s.session_id !== sessionId)) + if (currentSession?.session_id === sessionId) { + setCurrentSession(null) + setMessages([]) + } + Message.success('会话已删除') + } catch (error) { + Message.error('删除失败') + console.error(error) + } + }, + }) + } + + const handleEditTitle = (session) => { + setEditingSessionId(session.session_id) + setEditingTitle(session.title || `对话 ${session.session_id}`) + } + + const handleSaveTitle = async () => { + if (!editingSessionId || !editingTitle.trim()) { + return + } + try { + await updateChatSessionTitle(editingSessionId, editingTitle) + setSessions(sessions.map(s => + s.session_id === editingSessionId + ? { ...s, title: editingTitle } + : s + )) + setEditingSessionId(null) + Message.success('标题已更新') + } catch (error) { + Message.error('更新失败') + console.error(error) + } + } + + const currentModel = llmConfigs.find(m => m.config_id === selectedModel) + + return ( + + +
+
+
项目
+ ({ + label: m.model_name, + value: m.config_id + }))} + /> +
+ + +
+ + + +
+ {sessionLoading ? ( + + ) : sessions.length === 0 ? ( + + ) : ( + sessions.map(session => ( + { + setCurrentSession(session) + fetchMessages(session.session_id) + }} + style={{ + marginBottom: '8px', + cursor: 'pointer', + background: currentSession?.session_id === session.session_id ? '#e6f7ff' : '#fff', + border: currentSession?.session_id === session.session_id ? '1px solid #1890ff' : '1px solid #d9d9d9', + }} + > +
+
+ {editingSessionId === session.session_id ? ( + setEditingTitle(e.target.value)} + size="small" + onBlur={handleSaveTitle} + onPressEnter={handleSaveTitle} + autoFocus + /> + ) : ( +
+ {session.title || `对话 ${session.session_id}`} +
+ )} +
+ {new Date(session.created_at).toLocaleDateString()} +
+
+ + { + e.stopPropagation() + handleEditTitle(session) + }} + /> + { + e.stopPropagation() + handleDeleteSession(session.session_id) + }} + /> + +
+ + )) + )} +
+ + + + {!currentSession ? ( + + ) : ( + <> +
+
+
+ {currentSession.title || `对话 ${currentSession.session_id}`} +
+ {currentModel && ( + {currentModel.model_name} + )} +
+
+ +
+ {messageLoading ? ( + + ) : messages.length === 0 ? ( + + ) : ( + messages.map((msg) => ( +
+
+ {msg.content} +
+
+ )) + )} +
+
+ +
+ setInputValue(e.target.value)} + onPressEnter={handleSendMessage} + disabled={messageLoading} + autoFocus + /> + +
+ + )} + + + ) +} + +export default Chat