AstrBot/astrbot/dashboard/routes/knowledge_base.py

"""知识库管理 API 路由"""

import asyncio
import os
import traceback
import uuid
from typing import Any

import aiofiles
from quart import request

from astrbot.core import logger
from astrbot.core.core_lifecycle import AstrBotCoreLifecycle
from astrbot.core.provider.provider import EmbeddingProvider, RerankProvider
from astrbot.core.utils.astrbot_path import get_astrbot_temp_path

from ..utils import generate_tsne_visualization
from .route import Response, Route, RouteContext


class KnowledgeBaseRoute(Route):
    """知识库管理路由

    提供知识库、文档、检索、会话配置等 API 接口
    """

    def __init__(
        self,
        context: RouteContext,
        core_lifecycle: AstrBotCoreLifecycle,
    ) -> None:
        super().__init__(context)
        self.core_lifecycle = core_lifecycle
        self.kb_manager = None  # 延迟初始化
        self.kb_db = None
        self.session_config_db = None  # 会话配置数据库
        self.retrieval_manager = None
        self.upload_progress = {}  # 存储上传进度 {task_id: {status, file_index, file_total, stage, current, total}}
        self.upload_tasks = {}  # 存储后台上传任务 {task_id: {"status", "result", "error"}}

        # 注册路由
        self.routes = {
            # 知识库管理
            "/kb/list": ("GET", self.list_kbs),
            "/kb/create": ("POST", self.create_kb),
            "/kb/get": ("GET", self.get_kb),
            "/kb/update": ("POST", self.update_kb),
            "/kb/delete": ("POST", self.delete_kb),
            "/kb/stats": ("GET", self.get_kb_stats),
            # 文档管理
            "/kb/document/list": ("GET", self.list_documents),
            "/kb/document/upload": ("POST", self.upload_document),
            "/kb/document/import": ("POST", self.import_documents),
            "/kb/document/upload/url": ("POST", self.upload_document_from_url),
            "/kb/document/upload/progress": ("GET", self.get_upload_progress),
            "/kb/document/get": ("GET", self.get_document),
            "/kb/document/delete": ("POST", self.delete_document),
            # # 块管理
            "/kb/chunk/list": ("GET", self.list_chunks),
            "/kb/chunk/delete": ("POST", self.delete_chunk),
            # # 多媒体管理
            # "/kb/media/list": ("GET", self.list_media),
            # "/kb/media/delete": ("POST", self.delete_media),
            # 检索
            "/kb/retrieve": ("POST", self.retrieve),
        }
        self.register_routes()

    def _get_kb_manager(self):
        return self.core_lifecycle.kb_manager

    def _init_task(self, task_id: str, status: str = "pending") -> None:
        self.upload_tasks[task_id] = {
            "status": status,
            "result": None,
            "error": None,
        }

    def _set_task_result(
        self, task_id: str, status: str, result: Any = None, error: str | None = None
    ) -> None:
        self.upload_tasks[task_id] = {
            "status": status,
            "result": result,
            "error": error,
        }
        if task_id in self.upload_progress:
            self.upload_progress[task_id]["status"] = status

    def _update_progress(
        self,
        task_id: str,
        *,
        status: str | None = None,
        file_index: int | None = None,
        file_name: str | None = None,
        stage: str | None = None,
        current: int | None = None,
        total: int | None = None,
    ) -> None:
        if task_id not in self.upload_progress:
            return
        p = self.upload_progress[task_id]
        if status is not None:
            p["status"] = status
        if file_index is not None:
            p["file_index"] = file_index
        if file_name is not None:
            p["file_name"] = file_name
        if stage is not None:
            p["stage"] = stage
        if current is not None:
            p["current"] = current
        if total is not None:
            p["total"] = total

    def _make_progress_callback(self, task_id: str, file_idx: int, file_name: str):
        async def _callback(stage: str, current: int, total: int) -> None:
            self._update_progress(
                task_id,
                status="processing",
                file_index=file_idx,
                file_name=file_name,
                stage=stage,
                current=current,
                total=total,
            )

        return _callback

    async def _background_upload_task(
        self,
        task_id: str,
        kb_helper,
        files_to_upload: list,
        chunk_size: int,
        chunk_overlap: int,
        batch_size: int,
        tasks_limit: int,
        max_retries: int,
    ) -> None:
        """后台上传任务"""
        try:
            # 初始化任务状态
            self._init_task(task_id, status="processing")
            self.upload_progress[task_id] = {
                "status": "processing",
                "file_index": 0,
                "file_total": len(files_to_upload),
                "stage": "waiting",
                "current": 0,
                "total": 100,
            }

            uploaded_docs = []
            failed_docs = []

            for file_idx, file_info in enumerate(files_to_upload):
                try:
                    # 更新整体进度
                    self._update_progress(
                        task_id,
                        status="processing",
                        file_index=file_idx,
                        file_name=file_info["file_name"],
                        stage="parsing",
                        current=0,
                        total=100,
                    )

                    # 创建进度回调函数
                    progress_callback = self._make_progress_callback(
                        task_id, file_idx, file_info["file_name"]
                    )

                    doc = await kb_helper.upload_document(
                        file_name=file_info["file_name"],
                        file_content=file_info["file_content"],
                        file_type=file_info["file_type"],
                        chunk_size=chunk_size,
                        chunk_overlap=chunk_overlap,
                        batch_size=batch_size,
                        tasks_limit=tasks_limit,
                        max_retries=max_retries,
                        progress_callback=progress_callback,
                    )

                    uploaded_docs.append(doc.model_dump())
                except Exception as e:
                    logger.error(f"上传文档 {file_info['file_name']} 失败: {e}")
                    failed_docs.append(
                        {"file_name": file_info["file_name"], "error": str(e)},
                    )

            # 更新任务完成状态
            result = {
                "task_id": task_id,
                "uploaded": uploaded_docs,
                "failed": failed_docs,
                "total": len(files_to_upload),
                "success_count": len(uploaded_docs),
                "failed_count": len(failed_docs),
            }

            self._set_task_result(task_id, "completed", result=result)

        except Exception as e:
            logger.error(f"后台上传任务 {task_id} 失败: {e}")
            logger.error(traceback.format_exc())
            self._set_task_result(task_id, "failed", error=str(e))

    async def _background_import_task(
        self,
        task_id: str,
        kb_helper,
        documents: list,
        batch_size: int,
        tasks_limit: int,
        max_retries: int,
    ) -> None:
        """后台导入预切片文档任务"""
        try:
            # 初始化任务状态
            self._init_task(task_id, status="processing")
            self.upload_progress[task_id] = {
                "status": "processing",
                "file_index": 0,
                "file_total": len(documents),
                "stage": "waiting",
                "current": 0,
                "total": 100,
            }

            uploaded_docs = []
            failed_docs = []

            for file_idx, doc_info in enumerate(documents):
                file_name = doc_info.get("file_name", f"imported_doc_{file_idx}")
                chunks = doc_info.get("chunks", [])

                try:
                    # 更新整体进度
                    self._update_progress(
                        task_id,
                        status="processing",
                        file_index=file_idx,
                        file_name=file_name,
                        stage="importing",
                        current=0,
                        total=100,
                    )

                    # 创建进度回调函数
                    progress_callback = self._make_progress_callback(
                        task_id, file_idx, file_name
                    )

                    # 调用 upload_document，传入 pre_chunked_text
                    doc = await kb_helper.upload_document(
                        file_name=file_name,
                        file_content=None,  # 预切片模式下不需要原始内容
                        file_type=doc_info.get("file_type")
                        or (
                            file_name.rsplit(".", 1)[-1].lower()
                            if "." in file_name
                            else "txt"
                        ),
                        batch_size=batch_size,
                        tasks_limit=tasks_limit,
                        max_retries=max_retries,
                        progress_callback=progress_callback,
                        pre_chunked_text=chunks,
                    )

                    uploaded_docs.append(doc.model_dump())
                except Exception as e:
                    logger.error(f"导入文档 {file_name} 失败: {e}")
                    failed_docs.append(
                        {"file_name": file_name, "error": str(e)},
                    )

            # 更新任务完成状态
            result = {
                "task_id": task_id,
                "uploaded": uploaded_docs,
                "failed": failed_docs,
                "total": len(documents),
                "success_count": len(uploaded_docs),
                "failed_count": len(failed_docs),
            }

            self._set_task_result(task_id, "completed", result=result)

        except Exception as e:
            logger.error(f"后台导入任务 {task_id} 失败: {e}")
            logger.error(traceback.format_exc())
            self._set_task_result(task_id, "failed", error=str(e))

    async def list_kbs(self):
        """获取知识库列表

        Query 参数:
        - page: 页码 (默认 1)
        - page_size: 每页数量 (默认 20)
        - refresh_stats: 是否刷新统计信息 (默认 false，首次加载时可设为 true)
        """
        try:
            kb_manager = self._get_kb_manager()
            page = request.args.get("page", 1, type=int)
            page_size = request.args.get("page_size", 20, type=int)

            kbs = await kb_manager.list_kbs()

            # 转换为字典列表
            kb_list = []
            for kb in kbs:
                kb_list.append(kb.model_dump())

            return (
                Response()
                .ok({"items": kb_list, "page": page, "page_size": page_size})
                .__dict__
            )
        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取知识库列表失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取知识库列表失败: {e!s}").__dict__

    async def create_kb(self):
        """创建知识库

        Body:
        - kb_name: 知识库名称 (必填)
        - description: 描述 (可选)
        - emoji: 图标 (可选)
        - embedding_provider_id: 嵌入模型提供商ID (可选)
        - rerank_provider_id: 重排序模型提供商ID (可选)
        - chunk_size: 分块大小 (可选, 默认512)
        - chunk_overlap: 块重叠大小 (可选, 默认50)
        - top_k_dense: 密集检索数量 (可选, 默认50)
        - top_k_sparse: 稀疏检索数量 (可选, 默认50)
        - top_m_final: 最终返回数量 (可选, 默认5)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json
            kb_name = data.get("kb_name")
            if not kb_name:
                return Response().error("知识库名称不能为空").__dict__

            description = data.get("description")
            emoji = data.get("emoji")
            embedding_provider_id = data.get("embedding_provider_id")
            rerank_provider_id = data.get("rerank_provider_id")
            chunk_size = data.get("chunk_size")
            chunk_overlap = data.get("chunk_overlap")
            top_k_dense = data.get("top_k_dense")
            top_k_sparse = data.get("top_k_sparse")
            top_m_final = data.get("top_m_final")

            # pre-check embedding dim
            if not embedding_provider_id:
                return Response().error("缺少参数 embedding_provider_id").__dict__
            prv = await kb_manager.provider_manager.get_provider_by_id(
                embedding_provider_id,
            )  # type: ignore
            if not prv or not isinstance(prv, EmbeddingProvider):
                return (
                    Response().error(f"嵌入模型不存在或类型错误({type(prv)})").__dict__
                )
            try:
                vec = await prv.get_embedding("astrbot")
                if len(vec) != prv.get_dim():
                    raise ValueError(
                        f"嵌入向量维度不匹配，实际是 {len(vec)}，然而配置是 {prv.get_dim()}",
                    )
            except Exception as e:
                return Response().error(f"测试嵌入模型失败: {e!s}").__dict__
            # pre-check rerank
            if rerank_provider_id:
                rerank_prv: RerankProvider = (
                    await kb_manager.provider_manager.get_provider_by_id(
                        rerank_provider_id,
                    )
                )  # type: ignore
                if not rerank_prv:
                    return Response().error("重排序模型不存在").__dict__
                # 检查重排序模型可用性
                try:
                    res = await rerank_prv.rerank(
                        query="astrbot",
                        documents=["astrbot knowledge base"],
                    )
                    if not res:
                        raise ValueError("重排序模型返回结果异常")
                except Exception as e:
                    return (
                        Response()
                        .error(f"测试重排序模型失败: {e!s}，请检查平台日志输出。")
                        .__dict__
                    )

            kb_helper = await kb_manager.create_kb(
                kb_name=kb_name,
                description=description,
                emoji=emoji,
                embedding_provider_id=embedding_provider_id,
                rerank_provider_id=rerank_provider_id,
                chunk_size=chunk_size,
                chunk_overlap=chunk_overlap,
                top_k_dense=top_k_dense,
                top_k_sparse=top_k_sparse,
                top_m_final=top_m_final,
            )
            kb = kb_helper.kb

            return Response().ok(kb.model_dump(), "创建知识库成功").__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"创建知识库失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"创建知识库失败: {e!s}").__dict__

    async def get_kb(self):
        """获取知识库详情

        Query 参数:
        - kb_id: 知识库 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            kb_id = request.args.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__
            kb = kb_helper.kb

            return Response().ok(kb.model_dump()).__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取知识库详情失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取知识库详情失败: {e!s}").__dict__

    async def update_kb(self):
        """更新知识库

        Body:
        - kb_id: 知识库 ID (必填)
        - kb_name: 新的知识库名称 (可选)
        - description: 新的描述 (可选)
        - emoji: 新的图标 (可选)
        - embedding_provider_id: 新的嵌入模型提供商ID (可选)
        - rerank_provider_id: 新的重排序模型提供商ID (可选)
        - chunk_size: 分块大小 (可选)
        - chunk_overlap: 块重叠大小 (可选)
        - top_k_dense: 密集检索数量 (可选)
        - top_k_sparse: 稀疏检索数量 (可选)
        - top_m_final: 最终返回数量 (可选)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id = data.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            kb_name = data.get("kb_name")
            description = data.get("description")
            emoji = data.get("emoji")
            embedding_provider_id = data.get("embedding_provider_id")
            rerank_provider_id = data.get("rerank_provider_id")
            chunk_size = data.get("chunk_size")
            chunk_overlap = data.get("chunk_overlap")
            top_k_dense = data.get("top_k_dense")
            top_k_sparse = data.get("top_k_sparse")
            top_m_final = data.get("top_m_final")

            # 检查是否至少提供了一个更新字段
            if all(
                v is None
                for v in [
                    kb_name,
                    description,
                    emoji,
                    embedding_provider_id,
                    rerank_provider_id,
                    chunk_size,
                    chunk_overlap,
                    top_k_dense,
                    top_k_sparse,
                    top_m_final,
                ]
            ):
                return Response().error("至少需要提供一个更新字段").__dict__

            kb_helper = await kb_manager.update_kb(
                kb_id=kb_id,
                kb_name=kb_name,
                description=description,
                emoji=emoji,
                embedding_provider_id=embedding_provider_id,
                rerank_provider_id=rerank_provider_id,
                chunk_size=chunk_size,
                chunk_overlap=chunk_overlap,
                top_k_dense=top_k_dense,
                top_k_sparse=top_k_sparse,
                top_m_final=top_m_final,
            )

            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            kb = kb_helper.kb
            return Response().ok(kb.model_dump(), "更新知识库成功").__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"更新知识库失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"更新知识库失败: {e!s}").__dict__

    async def delete_kb(self):
        """删除知识库

        Body:
        - kb_id: 知识库 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id = data.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            success = await kb_manager.delete_kb(kb_id)
            if not success:
                return Response().error("知识库不存在").__dict__

            return Response().ok(message="删除知识库成功").__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"删除知识库失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"删除知识库失败: {e!s}").__dict__

    async def get_kb_stats(self):
        """获取知识库统计信息

        Query 参数:
        - kb_id: 知识库 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            kb_id = request.args.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__
            kb = kb_helper.kb

            stats = {
                "kb_id": kb.kb_id,
                "kb_name": kb.kb_name,
                "doc_count": kb.doc_count,
                "chunk_count": kb.chunk_count,
                "created_at": kb.created_at.isoformat(),
                "updated_at": kb.updated_at.isoformat(),
            }

            return Response().ok(stats).__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取知识库统计失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取知识库统计失败: {e!s}").__dict__

    # ===== 文档管理 API =====

    async def list_documents(self):
        """获取文档列表

        Query 参数:
        - kb_id: 知识库 ID (必填)
        - page: 页码 (默认 1)
        - page_size: 每页数量 (默认 20)
        """
        try:
            kb_manager = self._get_kb_manager()
            kb_id = request.args.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__
            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            page = request.args.get("page", 1, type=int)
            page_size = request.args.get("page_size", 100, type=int)

            offset = (page - 1) * page_size
            limit = page_size

            doc_list = await kb_helper.list_documents(offset=offset, limit=limit)

            doc_list = [doc.model_dump() for doc in doc_list]

            return (
                Response()
                .ok({"items": doc_list, "page": page, "page_size": page_size})
                .__dict__
            )

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取文档列表失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取文档列表失败: {e!s}").__dict__

    async def upload_document(self):
        """上传文档

        支持两种方式:
        1. multipart/form-data 文件上传（支持多文件，最多10个）
        2. JSON 格式 base64 编码上传（支持多文件，最多10个）

        Form Data (multipart/form-data):
        - kb_id: 知识库 ID (必填)
        - file: 文件对象 (必填，可多个，字段名为 file, file1, file2, ... 或 files[])

        JSON Body (application/json):
        - kb_id: 知识库 ID (必填)
        - files: 文件数组 (必填)
          - file_name: 文件名 (必填)
          - file_content: base64 编码的文件内容 (必填)

        返回:
        - task_id: 任务ID，用于查询上传进度和结果
        """
        try:
            kb_manager = self._get_kb_manager()

            # 检查 Content-Type
            content_type = request.content_type
            kb_id = None
            chunk_size = None
            chunk_overlap = None
            batch_size = 32
            tasks_limit = 3
            max_retries = 3
            files_to_upload = []  # 存储待上传的文件信息列表

            if content_type and "multipart/form-data" not in content_type:
                return (
                    Response().error("Content-Type 须为 multipart/form-data").__dict__
                )
            form_data = await request.form
            files = await request.files

            kb_id = form_data.get("kb_id")
            chunk_size = int(form_data.get("chunk_size", 512))
            chunk_overlap = int(form_data.get("chunk_overlap", 50))
            batch_size = int(form_data.get("batch_size", 32))
            tasks_limit = int(form_data.get("tasks_limit", 3))
            max_retries = int(form_data.get("max_retries", 3))
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            # 收集所有文件
            file_list = []
            # 支持 file, file1, file2, ... 或 files[] 格式
            for key in files.keys():
                if key == "file" or key.startswith("file") or key == "files[]":
                    file_items = files.getlist(key)
                    file_list.extend(file_items)

            if not file_list:
                return Response().error("缺少文件").__dict__

            # 限制文件数量
            if len(file_list) > 10:
                return Response().error("最多只能上传10个文件").__dict__

            # 处理每个文件
            for file in file_list:
                file_name = file.filename

                # 保存到临时文件
                temp_file_path = os.path.join(
                    get_astrbot_temp_path(),
                    f"kb_upload_{uuid.uuid4()}_{file_name}",
                )
                await file.save(temp_file_path)

                try:
                    # 异步读取文件内容
                    async with aiofiles.open(temp_file_path, "rb") as f:
                        file_content = await f.read()

                    # 提取文件类型
                    file_type = (
                        file_name.rsplit(".", 1)[-1].lower() if "." in file_name else ""
                    )

                    files_to_upload.append(
                        {
                            "file_name": file_name,
                            "file_content": file_content,
                            "file_type": file_type,
                        },
                    )
                finally:
                    # 清理临时文件
                    if os.path.exists(temp_file_path):
                        os.remove(temp_file_path)

            # 获取知识库
            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            # 生成任务ID
            task_id = str(uuid.uuid4())

            # 初始化任务状态
            self._init_task(task_id, status="pending")

            # 启动后台任务
            asyncio.create_task(
                self._background_upload_task(
                    task_id=task_id,
                    kb_helper=kb_helper,
                    files_to_upload=files_to_upload,
                    chunk_size=chunk_size,
                    chunk_overlap=chunk_overlap,
                    batch_size=batch_size,
                    tasks_limit=tasks_limit,
                    max_retries=max_retries,
                ),
            )

            return (
                Response()
                .ok(
                    {
                        "task_id": task_id,
                        "file_count": len(files_to_upload),
                        "message": "task created, processing in background",
                    },
                )
                .__dict__
            )

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"上传文档失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"上传文档失败: {e!s}").__dict__

    def _validate_import_request(self, data: dict):
        kb_id = data.get("kb_id")
        if not kb_id:
            raise ValueError("缺少参数 kb_id")

        documents = data.get("documents")
        if not documents or not isinstance(documents, list):
            raise ValueError("缺少参数 documents 或格式错误")

        for doc in documents:
            if "file_name" not in doc or "chunks" not in doc:
                raise ValueError("文档格式错误，必须包含 file_name 和 chunks")
            if not isinstance(doc["chunks"], list):
                raise ValueError("chunks 必须是列表")
            if not all(
                isinstance(chunk, str) and chunk.strip() for chunk in doc["chunks"]
            ):
                raise ValueError("chunks 必须是非空字符串列表")

        batch_size = data.get("batch_size", 32)
        tasks_limit = data.get("tasks_limit", 3)
        max_retries = data.get("max_retries", 3)
        return kb_id, documents, batch_size, tasks_limit, max_retries

    async def import_documents(self):
        """导入预切片文档

        Body:
        - kb_id: 知识库 ID (必填)
        - documents: 文档列表 (必填)
            - file_name: 文件名 (必填)
            - chunks: 切片列表 (必填, list[str])
            - file_type: 文件类型 (可选, 默认从文件名推断或为 txt)
        - batch_size: 批处理大小 (可选, 默认32)
        - tasks_limit: 并发任务限制 (可选, 默认3)
        - max_retries: 最大重试次数 (可选, 默认3)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id, documents, batch_size, tasks_limit, max_retries = (
                self._validate_import_request(data)
            )

            # 获取知识库
            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            # 生成任务ID
            task_id = str(uuid.uuid4())

            # 初始化任务状态
            self._init_task(task_id, status="pending")

            # 启动后台任务
            asyncio.create_task(
                self._background_import_task(
                    task_id=task_id,
                    kb_helper=kb_helper,
                    documents=documents,
                    batch_size=batch_size,
                    tasks_limit=tasks_limit,
                    max_retries=max_retries,
                ),
            )

            return (
                Response()
                .ok(
                    {
                        "task_id": task_id,
                        "doc_count": len(documents),
                        "message": "import task created, processing in background",
                    },
                )
                .__dict__
            )

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"导入文档失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"导入文档失败: {e!s}").__dict__

    async def get_upload_progress(self):
        """获取上传进度和结果

        Query 参数:
        - task_id: 任务 ID (必填)

        返回状态:
        - pending: 任务待处理
        - processing: 任务处理中
        - completed: 任务完成
        - failed: 任务失败
        """
        try:
            task_id = request.args.get("task_id")
            if not task_id:
                return Response().error("缺少参数 task_id").__dict__

            # 检查任务是否存在
            if task_id not in self.upload_tasks:
                return Response().error("找不到该任务").__dict__

            task_info = self.upload_tasks[task_id]
            status = task_info["status"]

            # 构建返回数据
            response_data = {
                "task_id": task_id,
                "status": status,
            }

            # 如果任务正在处理，返回进度信息
            if status == "processing" and task_id in self.upload_progress:
                response_data["progress"] = self.upload_progress[task_id]

            # 如果任务完成，返回结果
            if status == "completed":
                response_data["result"] = task_info["result"]
                # 清理已完成的任务
                # del self.upload_tasks[task_id]
                # if task_id in self.upload_progress:
                #     del self.upload_progress[task_id]

            # 如果任务失败，返回错误信息
            if status == "failed":
                response_data["error"] = task_info["error"]

            return Response().ok(response_data).__dict__

        except Exception as e:
            logger.error(f"获取上传进度失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取上传进度失败: {e!s}").__dict__

    async def get_document(self):
        """获取文档详情

        Query 参数:
        - doc_id: 文档 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            kb_id = request.args.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__
            doc_id = request.args.get("doc_id")
            if not doc_id:
                return Response().error("缺少参数 doc_id").__dict__
            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            doc = await kb_helper.get_document(doc_id)
            if not doc:
                return Response().error("文档不存在").__dict__

            return Response().ok(doc.model_dump()).__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取文档详情失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取文档详情失败: {e!s}").__dict__

    async def delete_document(self):
        """删除文档

        Body:
        - kb_id: 知识库 ID (必填)
        - doc_id: 文档 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id = data.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__
            doc_id = data.get("doc_id")
            if not doc_id:
                return Response().error("缺少参数 doc_id").__dict__

            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            await kb_helper.delete_document(doc_id)
            return Response().ok(message="删除文档成功").__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"删除文档失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"删除文档失败: {e!s}").__dict__

    async def delete_chunk(self):
        """删除文本块

        Body:
        - kb_id: 知识库 ID (必填)
        - chunk_id: 块 ID (必填)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id = data.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__
            chunk_id = data.get("chunk_id")
            if not chunk_id:
                return Response().error("缺少参数 chunk_id").__dict__
            doc_id = data.get("doc_id")
            if not doc_id:
                return Response().error("缺少参数 doc_id").__dict__

            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            await kb_helper.delete_chunk(chunk_id, doc_id)
            return Response().ok(message="删除文本块成功").__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"删除文本块失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"删除文本块失败: {e!s}").__dict__

    async def list_chunks(self):
        """获取块列表

        Query 参数:
        - kb_id: 知识库 ID (必填)
        - page: 页码 (默认 1)
        - page_size: 每页数量 (默认 20)
        """
        try:
            kb_manager = self._get_kb_manager()
            kb_id = request.args.get("kb_id")
            doc_id = request.args.get("doc_id")
            page = request.args.get("page", 1, type=int)
            page_size = request.args.get("page_size", 100, type=int)
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__
            if not doc_id:
                return Response().error("缺少参数 doc_id").__dict__
            kb_helper = await kb_manager.get_kb(kb_id)
            offset = (page - 1) * page_size
            limit = page_size
            if not kb_helper:
                return Response().error("知识库不存在").__dict__
            chunk_list = await kb_helper.get_chunks_by_doc_id(
                doc_id=doc_id,
                offset=offset,
                limit=limit,
            )
            return (
                Response()
                .ok(
                    data={
                        "items": chunk_list,
                        "page": page,
                        "page_size": page_size,
                        "total": await kb_helper.get_chunk_count_by_doc_id(doc_id),
                    },
                )
                .__dict__
            )
        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"获取块列表失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"获取块列表失败: {e!s}").__dict__

    # ===== 检索 API =====

    async def retrieve(self):
        """检索知识库

        Body:
        - query: 查询文本 (必填)
        - kb_ids: 知识库 ID 列表 (必填)
        - top_k: 返回结果数量 (可选, 默认 5)
        - debug: 是否启用调试模式，返回 t-SNE 可视化图片 (可选, 默认 False)
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            query = data.get("query")
            kb_names = data.get("kb_names")
            debug = data.get("debug", False)

            if not query:
                return Response().error("缺少参数 query").__dict__
            if not kb_names or not isinstance(kb_names, list):
                return Response().error("缺少参数 kb_names 或格式错误").__dict__

            top_k = data.get("top_k", 5)

            results = await kb_manager.retrieve(
                query=query,
                kb_names=kb_names,
                top_m_final=top_k,
            )
            result_list = []
            if results:
                result_list = results["results"]

            response_data = {
                "results": result_list,
                "total": len(result_list),
                "query": query,
            }

            # Debug 模式：生成 t-SNE 可视化
            if debug:
                try:
                    img_base64 = await generate_tsne_visualization(
                        query,
                        kb_names,
                        kb_manager,
                    )
                    if img_base64:
                        response_data["visualization"] = img_base64
                except Exception as e:
                    logger.error(f"生成 t-SNE 可视化失败: {e}")
                    logger.error(traceback.format_exc())
                    response_data["visualization_error"] = str(e)

            return Response().ok(response_data).__dict__

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"检索失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"检索失败: {e!s}").__dict__

    async def upload_document_from_url(self):
        """从 URL 上传文档

        Body:
        - kb_id: 知识库 ID (必填)
        - url: 要提取内容的网页 URL (必填)
        - chunk_size: 分块大小 (可选, 默认512)
        - chunk_overlap: 块重叠大小 (可选, 默认50)
        - batch_size: 批处理大小 (可选, 默认32)
        - tasks_limit: 并发任务限制 (可选, 默认3)
        - max_retries: 最大重试次数 (可选, 默认3)

        返回:
        - task_id: 任务ID，用于查询上传进度和结果
        """
        try:
            kb_manager = self._get_kb_manager()
            data = await request.json

            kb_id = data.get("kb_id")
            if not kb_id:
                return Response().error("缺少参数 kb_id").__dict__

            url = data.get("url")
            if not url:
                return Response().error("缺少参数 url").__dict__

            chunk_size = data.get("chunk_size", 512)
            chunk_overlap = data.get("chunk_overlap", 50)
            batch_size = data.get("batch_size", 32)
            tasks_limit = data.get("tasks_limit", 3)
            max_retries = data.get("max_retries", 3)
            enable_cleaning = data.get("enable_cleaning", False)
            cleaning_provider_id = data.get("cleaning_provider_id")

            # 获取知识库
            kb_helper = await kb_manager.get_kb(kb_id)
            if not kb_helper:
                return Response().error("知识库不存在").__dict__

            # 生成任务ID
            task_id = str(uuid.uuid4())

            # 初始化任务状态
            self._init_task(task_id, status="pending")

            # 启动后台任务
            asyncio.create_task(
                self._background_upload_from_url_task(
                    task_id=task_id,
                    kb_helper=kb_helper,
                    url=url,
                    chunk_size=chunk_size,
                    chunk_overlap=chunk_overlap,
                    batch_size=batch_size,
                    tasks_limit=tasks_limit,
                    max_retries=max_retries,
                    enable_cleaning=enable_cleaning,
                    cleaning_provider_id=cleaning_provider_id,
                ),
            )

            return (
                Response()
                .ok(
                    {
                        "task_id": task_id,
                        "url": url,
                        "message": "URL upload task created, processing in background",
                    },
                )
                .__dict__
            )

        except ValueError as e:
            return Response().error(str(e)).__dict__
        except Exception as e:
            logger.error(f"从URL上传文档失败: {e}")
            logger.error(traceback.format_exc())
            return Response().error(f"从URL上传文档失败: {e!s}").__dict__

    async def _background_upload_from_url_task(
        self,
        task_id: str,
        kb_helper,
        url: str,
        chunk_size: int,
        chunk_overlap: int,
        batch_size: int,
        tasks_limit: int,
        max_retries: int,
        enable_cleaning: bool,
        cleaning_provider_id: str | None,
    ) -> None:
        """后台上传URL任务"""
        try:
            # 初始化任务状态
            self._init_task(task_id, status="processing")
            self.upload_progress[task_id] = {
                "status": "processing",
                "file_index": 0,
                "file_total": 1,
                "file_name": f"URL: {url}",
                "stage": "extracting",
                "current": 0,
                "total": 100,
            }

            # 创建进度回调函数
            progress_callback = self._make_progress_callback(task_id, 0, f"URL: {url}")

            # 上传文档
            doc = await kb_helper.upload_from_url(
                url=url,
                chunk_size=chunk_size,
                chunk_overlap=chunk_overlap,
                batch_size=batch_size,
                tasks_limit=tasks_limit,
                max_retries=max_retries,
                progress_callback=progress_callback,
                enable_cleaning=enable_cleaning,
                cleaning_provider_id=cleaning_provider_id,
            )

            # 更新任务完成状态
            result = {
                "task_id": task_id,
                "uploaded": [doc.model_dump()],
                "failed": [],
                "total": 1,
                "success_count": 1,
                "failed_count": 0,
            }

            self._set_task_result(task_id, "completed", result=result)

        except Exception as e:
            logger.error(f"后台上传URL任务 {task_id} 失败: {e}")
            logger.error(traceback.format_exc())
            self._set_task_result(task_id, "failed", error=str(e))