assist/src/knowledge_base/knowledge_manager.py

import json
import logging
from typing import List, Dict, Optional, Any
from datetime import datetime
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sqlalchemy import func

from ..core.database import db_manager
from ..core.models import KnowledgeEntry, WorkOrder, Conversation
from ..core.llm_client import QwenClient

logger = logging.getLogger(__name__)

class KnowledgeManager:
    """知识库管理器"""
    
    def __init__(self):
        # 使用单例避免重复创建
        from ..core.component_singletons import component_singletons
        self.llm_client = component_singletons.get_llm_client()
        
        self.vectorizer = TfidfVectorizer(
            max_features=1000,
            stop_words=None,  # 不使用英文停用词，因为数据是中文
            ngram_range=(1, 2)
        )
        self._load_vectorizer()
    
    def _load_vectorizer(self):
        """加载向量化器"""
        try:
            with db_manager.get_session() as session:
                entries = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.is_active == True
                ).all()
                
                if entries:
                    texts = [entry.question + " " + entry.answer for entry in entries]
                    self.vectorizer.fit(texts)
                    logger.info(f"向量化器加载成功，包含 {len(entries)} 个条目")
        except Exception as e:
            logger.error(f"加载向量化器失败: {e}")
    
    def learn_from_work_order(self, work_order_id: int) -> bool:
        """从工单中学习知识"""
        try:
            with db_manager.get_session() as session:
                work_order = session.query(WorkOrder).filter(
                    WorkOrder.id == work_order_id
                ).first()
                
                if not work_order or not work_order.resolution:
                    return False
                
                # 提取问题和答案
                question = work_order.title + " " + work_order.description
                answer = work_order.resolution
                
                # 检查是否已存在相似条目
                existing_entry = self._find_similar_entry(question, session)
                
                if existing_entry:
                    # 更新现有条目
                    existing_entry.answer = answer
                    existing_entry.usage_count += 1
                    existing_entry.updated_at = datetime.now()
                    if work_order.satisfaction_score:
                        existing_entry.confidence_score = work_order.satisfaction_score
                else:
                    # 创建新条目
                    new_entry = KnowledgeEntry(
                        question=question,
                        answer=answer,
                        category=work_order.category,
                        confidence_score=work_order.satisfaction_score or 0.5,
                        usage_count=1
                    )
                    session.add(new_entry)
                
                session.commit()
                logger.info(f"从工单 {work_order_id} 学习知识成功")
                return True
                
        except Exception as e:
            logger.error(f"从工单学习知识失败: {e}")
            return False
    
    def _find_similar_entry(self, question: str, session) -> Optional[KnowledgeEntry]:
        """查找相似的知识库条目"""
        try:
            entries = session.query(KnowledgeEntry).filter(
                KnowledgeEntry.is_active == True
            ).all()
            
            if not entries:
                return None
            
            # 计算相似度
            texts = [entry.question for entry in entries]
            question_vector = self.vectorizer.transform([question])
            entry_vectors = self.vectorizer.transform(texts)
            
            similarities = cosine_similarity(question_vector, entry_vectors)[0]
            max_similarity_idx = np.argmax(similarities)
            
            if similarities[max_similarity_idx] > 0.8:  # 相似度阈值
                return entries[max_similarity_idx]
            
            return None
            
        except Exception as e:
            logger.error(f"查找相似条目失败: {e}")
            return None
    
    def search_knowledge(self, query: str, top_k: int = 3, verified_only: bool = True) -> List[Dict[str, Any]]:
        """搜索知识库"""
        try:
            with db_manager.get_session() as session:
                # 构建查询条件
                query_filter = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.is_active == True
                )
                
                # 如果只搜索已验证的知识库
                if verified_only:
                    query_filter = query_filter.filter(KnowledgeEntry.is_verified == True)
                
                entries = query_filter.all()
                # 若已验证为空，则回退到全部活跃条目
                if not entries and verified_only:
                    entries = session.query(KnowledgeEntry).filter(KnowledgeEntry.is_active == True).all()
                
                if not entries:
                    return []
                
                # 计算相似度
                texts = [entry.question + " " + entry.answer for entry in entries]

                # 确保向量器已训练
                try:
                    vocab_ok = hasattr(self.vectorizer, 'vocabulary_') and bool(self.vectorizer.vocabulary_)
                    if not vocab_ok:
                        self.vectorizer.fit(texts)
                    query_vector = self.vectorizer.transform([query])
                    entry_vectors = self.vectorizer.transform(texts)
                    similarities = cosine_similarity(query_vector, entry_vectors)[0]
                except Exception as vec_err:
                    logger.warning(f"TF-IDF搜索失败，回退到子串匹配: {vec_err}")
                    # 回退：子串匹配评分
                    similarities = []
                    q = query.strip()
                    for t in texts:
                        if not q:
                            similarities.append(0.0)
                        else:
                            score = 1.0 if q in t else 0.0
                            similarities.append(score)
                    similarities = np.array(similarities, dtype=float)
                
                # 获取top_k个最相似的条目
                top_indices = np.argsort(similarities)[-top_k:][::-1]
                
                results = []
                for idx in top_indices:
                    if similarities[idx] > 0.1:  # 最小相似度阈值
                        entry = entries[idx]
                        results.append({
                            "id": entry.id,
                            "question": entry.question,
                            "answer": entry.answer,
                            "category": entry.category,
                            "confidence_score": entry.confidence_score,
                            "similarity_score": float(similarities[idx]),
                            "usage_count": entry.usage_count,
                            "is_verified": entry.is_verified
                        })
                
                return results
                
        except Exception as e:
            logger.error(f"搜索知识库失败: {e}")
            return []
    
    def add_knowledge_entry(
        self,
        question: str,
        answer: str,
        category: str,
        confidence_score: float = 0.5,
        is_verified: bool = False
    ) -> bool:
        """添加知识库条目"""
        try:
            with db_manager.get_session() as session:
                entry = KnowledgeEntry(
                    question=question,
                    answer=answer,
                    category=category,
                    confidence_score=confidence_score,
                    usage_count=0,
                    is_verified=is_verified
                )
                session.add(entry)
                session.commit()
                
                # 重新训练向量化器
                self._load_vectorizer()
                
                logger.info(f"添加知识库条目成功: {question[:50]}...")
                return True
                
        except Exception as e:
            logger.error(f"添加知识库条目失败: {e}")
            return False
    
    def update_knowledge_entry(
        self,
        entry_id: int,
        question: str = None,
        answer: str = None,
        category: str = None,
        confidence_score: float = None
    ) -> bool:
        """更新知识库条目"""
        try:
            with db_manager.get_session() as session:
                entry = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.id == entry_id
                ).first()
                
                if not entry:
                    return False
                
                if question:
                    entry.question = question
                if answer:
                    entry.answer = answer
                if category:
                    entry.category = category
                if confidence_score is not None:
                    entry.confidence_score = confidence_score
                
                entry.updated_at = datetime.now()
                session.commit()
                
                logger.info(f"更新知识库条目成功: {entry_id}")
                return True
                
        except Exception as e:
            logger.error(f"更新知识库条目失败: {e}")
            return False
    
    def get_knowledge_entries(self, page: int = 1, per_page: int = 10) -> Dict[str, Any]:
        """获取知识库条目（分页）"""
        try:
            with db_manager.get_session() as session:
                # 计算偏移量
                offset = (page - 1) * per_page
                
                # 获取总数
                total = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.is_active == True
                ).count()
                
                # 获取分页数据
                entries = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.is_active == True
                ).order_by(KnowledgeEntry.created_at.desc()).offset(offset).limit(per_page).all()
                
                # 转换为字典格式
                knowledge_list = []
                for entry in entries:
                    knowledge_list.append({
                        "id": entry.id,
                        "question": entry.question,
                        "answer": entry.answer,
                        "category": entry.category,
                        "confidence_score": entry.confidence_score,
                        "usage_count": entry.usage_count,
                        "created_at": entry.created_at.isoformat() if entry.created_at else None,
                        "is_verified": getattr(entry, 'is_verified', False)  # 添加验证状态
                    })
                
                return {
                    "knowledge": knowledge_list,
                    "total": total,
                    "page": page,
                    "per_page": per_page,
                    "total_pages": (total + per_page - 1) // per_page
                }
        except Exception as e:
            logger.error(f"获取知识库条目失败: {e}")
            return {"knowledge": [], "total": 0, "page": 1, "per_page": per_page, "total_pages": 0}
    
    def verify_knowledge_entry(self, entry_id: int, verified_by: str = "admin") -> bool:
        """验证知识库条目"""
        try:
            with db_manager.get_session() as session:
                entry = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.id == entry_id
                ).first()
                
                if not entry:
                    return False
                
                entry.is_verified = True
                entry.verified_by = verified_by
                entry.verified_at = datetime.now()
                
                session.commit()
                logger.info(f"知识库条目验证成功: {entry_id}")
                return True
                
        except Exception as e:
            logger.error(f"验证知识库条目失败: {e}")
            return False
    
    def unverify_knowledge_entry(self, entry_id: int) -> bool:
        """取消验证知识库条目"""
        try:
            with db_manager.get_session() as session:
                entry = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.id == entry_id
                ).first()
                
                if not entry:
                    return False
                
                entry.is_verified = False
                entry.verified_by = None
                entry.verified_at = None
                
                session.commit()
                logger.info(f"知识库条目取消验证成功: {entry_id}")
                return True
                
        except Exception as e:
            logger.error(f"取消验证知识库条目失败: {e}")
            return False
    
    def delete_knowledge_entry(self, entry_id: int) -> bool:
        """删除知识库条目（软删除）"""
        try:
            with db_manager.get_session() as session:
                entry = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.id == entry_id
                ).first()
                
                if not entry:
                    logger.warning(f"知识库条目不存在: {entry_id}")
                    return False
                
                entry.is_active = False
                session.commit()
                
                # 重新训练向量化器（如果还有活跃条目）
                try:
                    self._load_vectorizer()
                except Exception as vectorizer_error:
                    logger.warning(f"重新加载向量化器失败: {vectorizer_error}")
                    # 即使向量化器加载失败，删除操作仍然成功
                
                logger.info(f"删除知识库条目成功: {entry_id}")
                return True
                
        except Exception as e:
            logger.error(f"删除知识库条目失败: {e}")
            return False
    
    def get_knowledge_stats(self) -> Dict[str, Any]:
        """获取知识库统计信息"""
        try:
            with db_manager.get_session() as session:
                total_entries = session.query(KnowledgeEntry).count()
                active_entries = session.query(KnowledgeEntry).filter(
                    KnowledgeEntry.is_active == True
                ).count()
                
                # 按类别统计
                category_stats = session.query(
                    KnowledgeEntry.category,
                    session.query(KnowledgeEntry).filter(
                        KnowledgeEntry.category == KnowledgeEntry.category
                    ).count()
                ).group_by(KnowledgeEntry.category).all()
                
                # 平均置信度
                avg_confidence = session.query(
                    func.avg(KnowledgeEntry.confidence_score)
                ).scalar() or 0.0
                
                return {
                    "total_entries": total_entries,
                    "active_entries": active_entries,
                    "category_distribution": dict(category_stats),
                    "average_confidence": float(avg_confidence)
                }
                
        except Exception as e:
            logger.error(f"获取知识库统计失败: {e}")
            return {}