feat: add token usage and top questions statistics endpoints

apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py

@@ -1,9 +1,7 @@
 # coding=utf-8
-import base64
+
-import mimetypes
 import time
 from functools import reduce
-from imghdr import what
 from typing import List, Dict
 
 from django.db.models import QuerySet
@@ -12,7 +10,6 @@ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AI
 from application.flow.i_step_node import NodeResult, INode
 from application.flow.step_node.video_understand_step_node.i_video_understand_node import IVideoUnderstandNode
 from knowledge.models import File
-from models_provider.impl.volcanic_engine_model_provider.model.image import get_video_format
 from models_provider.tools import get_model_instance_by_model_workspace_id
 
 

apps/application/serializers/application_stats.py

@@ -118,3 +118,37 @@ class ApplicationStatisticsSerializer(serializers.Serializer):
             days.append(current_date.strftime('%Y-%m-%d'))
             current_date += datetime.timedelta(days=1)
         return days
+
+    def get_token_usage_statistics(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        get_token_usage = native_search(
+            {'default_sql': QuerySet(model=get_dynamics_model(
+                {'application_chat.application_id': models.UUIDField(),
+                 'application_chat_record.create_time': models.DateTimeField()})).filter(
+                **{'application_chat.application_id': self.data.get('application_id'),
+                   'application_chat_record.create_time__gte': start_time,
+                   'application_chat_record.create_time__lte': end_time}
+            )},
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'get_token_usage.sql')))
+        return get_token_usage
+
+    def get_top_questions_statistics(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        get_top_questions = native_search(
+            {'default_sql': QuerySet(model=get_dynamics_model(
+                {'application_chat.application_id': models.UUIDField(),
+                 'application_chat_record.create_time': models.DateTimeField()})).filter(
+                **{'application_chat.application_id': self.data.get('application_id'),
+                   'application_chat_record.create_time__gte': start_time,
+                   'application_chat_record.create_time__lte': end_time}
+            )},
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'top_questions.sql')))
+        return get_top_questions

apps/application/sql/get_token_usage.sql

@@ -0,0 +1,12 @@
+
+SELECT
+    SUM(application_chat_record.message_tokens + application_chat_record.answer_tokens) as "token_usage",
+    COALESCE(application_chat.asker->>'username', '游客') as "username"
+FROM
+    application_chat_record application_chat_record
+    LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
+${default_sql}
+GROUP BY
+    COALESCE(application_chat.asker->>'username', '游客')
+ORDER BY
+    "token_usage" DESC

apps/application/sql/top_questions.sql

@@ -0,0 +1,11 @@
+SELECT COUNT(application_chat_record."id")                          AS chat_record_count,
+       COALESCE(application_chat.asker ->>'username', '游客') AS username
+FROM application_chat_record application_chat_record
+         LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
+    ${default_sql}
+GROUP BY
+    COALESCE (application_chat.asker->>'username', '游客')
+ORDER BY
+    chat_record_count DESC,
+    username ASC
+

apps/application/urls.py

@@ -13,6 +13,8 @@ urlpatterns = [
     path('workspace/<str:workspace_id>/application/<str:application_id>/publish', views.ApplicationAPI.Publish.as_view()),
     path('workspace/<str:workspace_id>/application/<str:application_id>/application_key', views.ApplicationKey.as_view()),
     path('workspace/<str:workspace_id>/application/<str:application_id>/application_stats', views.ApplicationStats.as_view()),
+    path('workspace/<str:workspace_id>/application/<str:application_id>/application_token_usage', views.ApplicationStats.TokenUsageStatistics.as_view()),
+    path('workspace/<str:workspace_id>/application/<str:application_id>/top_questions', views.ApplicationStats.TopQuestionsStatistics.as_view()),
     path('workspace/<str:workspace_id>/application/<str:application_id>/application_key/<str:api_key_id>', views.ApplicationKey.Operate.as_view()),
     path('workspace/<str:workspace_id>/application/<str:application_id>/export', views.ApplicationAPI.Export.as_view()),
     path('workspace/<str:workspace_id>/application/<str:application_id>/application_version', views.ApplicationVersionView.as_view()),

apps/application/views/application_stats.py

@@ -46,3 +46,58 @@ class ApplicationStats(APIView):
                                                   'end_time': request.query_params.get(
                                                       'end_time')
                                                   }).get_chat_record_aggregate_trend())
+
+    class TokenUsageStatistics(APIView):
+        authentication_classes = [TokenAuth]
+
+        # 应用的token使用统计 根据人的使用数排序
+        @extend_schema(
+            methods=['GET'],
+            description=_('Application token usage statistics'),
+            summary=_('Application token usage statistics'),
+            operation_id=_('Application token usage statistics'),  # type: ignore
+            parameters=ApplicationStatsAPI.get_parameters(),
+            responses=ApplicationStatsAPI.get_response(),
+            tags=[_('Application')]  # type: ignore
+        )
+        @has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
+                         PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
+                         ViewPermission([RoleConstants.USER.get_workspace_role()],
+                                        [PermissionConstants.APPLICATION.get_workspace_application_permission()],
+                                        CompareConstants.AND),
+                         RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
+        def get(self, request: Request, workspace_id: str, application_id: str):
+            return result.success(
+                ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
+                                                      'start_time': request.query_params.get(
+                                                          'start_time'),
+                                                      'end_time': request.query_params.get(
+                                                          'end_time')
+                                                      }).get_token_usage_statistics())
+
+    class TopQuestionsStatistics(APIView):
+        authentication_classes = [TokenAuth]
+        # 应用的top10问题统计
+        @extend_schema(
+            methods=['GET'],
+            description=_('Application top10 question statistics'),
+            summary=_('Application top10 question statistics'),
+            operation_id=_('Application top10 question statistics'),  # type: ignore
+            parameters=ApplicationStatsAPI.get_parameters(),
+            responses=ApplicationStatsAPI.get_response(),
+            tags=[_('Application')]  # type: ignore
+        )
+        @has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
+                         PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
+                         ViewPermission([RoleConstants.USER.get_workspace_role()],
+                                        [PermissionConstants.APPLICATION.get_workspace_application_permission()],
+                                        CompareConstants.AND),
+                         RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
+        def get(self, request: Request, workspace_id: str, application_id: str):
+            return result.success(
+                ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
+                                                      'start_time': request.query_params.get(
+                                                          'start_time'),
+                                                      'end_time': request.query_params.get(
+                                                          'end_time')
+                                                      }).get_top_questions_statistics())

apps/models_provider/impl/base_chat_open_ai.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+import base64
 from concurrent.futures import ThreadPoolExecutor
 from requests.exceptions import ConnectTimeout, ReadTimeout
 from typing import Dict, Optional, Any, Iterator, cast, Union, Sequence, Callable, Mapping
@@ -211,3 +212,20 @@ class BaseChatOpenAI(ChatOpenAI):
         self.usage_metadata = chat_result.response_metadata[
             'token_usage'] if 'token_usage' in chat_result.response_metadata else chat_result.usage_metadata
         return chat_result
+
+
+    def upload_file_and_get_url(self, file_stream, file_name):
+        """上传文件并获取文件URL"""
+        base64_video = base64.b64encode(file_stream).decode("utf-8")
+        video_format = get_video_format(file_name)
+        return f'data:{video_format};base64,{base64_video}'
+
+def get_video_format(file_name):
+    extension = file_name.split('.')[-1].lower()
+    format_map = {
+        'mp4': 'video/mp4',
+        'avi': 'video/avi',
+        'mov': 'video/mov',
+        'wmv': 'video/x-ms-wmv'
+    }
+    return format_map.get(extension, 'video/mp4')

apps/models_provider/impl/volcanic_engine_model_provider/model/image.py

@@ -25,20 +25,5 @@ class VolcanicEngineImage(MaxKBBaseModel, BaseChatOpenAI):
     def is_cache_model():
         return False
 
-    def upload_file_and_get_url(self, file_stream, file_name):
-        """上传文件并获取文件URL"""
-        base64_video = base64.b64encode(file_stream).decode("utf-8")
-        video_format = get_video_format(file_name)
-        return f'data:{video_format};base64,{base64_video}'
 
 
-
-def get_video_format(file_name):
-    extension = file_name.split('.')[-1].lower()
-    format_map = {
-        'mp4': 'video/mp4',
-        'avi': 'video/avi',
-        'mov': 'video/mov',
-        'wmv': 'video/x-ms-wmv'
-    }
-    return format_map.get(extension, 'video/mp4')

apps/models_provider/impl/zhipu_model_provider/model/image.py

@@ -3,7 +3,6 @@ from typing import Dict
 from models_provider.base_model_provider import MaxKBBaseModel
 from models_provider.impl.base_chat_open_ai import BaseChatOpenAI
 
-
 class ZhiPuImage(MaxKBBaseModel, BaseChatOpenAI):
 
     @staticmethod