from opentelemetry import trace

tracer = trace.get_tracer("ai-app")

def handle_user_query(query: str, user_id: str):
    with tracer.start_as_current_span("user-query") as root_span:
        root_span.set_attribute("user.id", user_id)
        root_span.set_attribute("query.text", query)
        
        # 1. RAG 检索
        with tracer.start_as_current_span("rag-retrieval") as rag_span:
            documents = retrieve_documents(query)
            rag_span.set_attribute("rag.documents.count", len(documents))
            rag_span.set_attribute("rag.documents.relevance", avg_relevance(documents))
        
        # 2. Prompt 构建
        with tracer.start_as_current_span("prompt-construction") as prompt_span:
            prompt = build_prompt(query, documents)
            prompt_span.set_attribute("prompt.tokens", count_tokens(prompt))
        
        # 3. LLM 调用
        with tracer.start_as_current_span("llm-inference") as llm_span:
            response = call_llm(prompt)
            llm_span.set_attribute("llm.model", "gpt-4")
            llm_span.set_attribute("llm.tokens.input", count_tokens(prompt))
            llm_span.set_attribute("llm.tokens.output", count_tokens(response))
            llm_span.set_attribute("llm.latency.first_token_ms", response.first_token_latency)
            llm_span.set_attribute("llm.latency.total_ms", response.total_latency)
        
        # 4. 后处理
        with tracer.start_as_current_span("post-processing") as post_span:
            filtered_response = filter_and_mask(response)
            post_span.set_attribute("post.pii_detected", filtered_response.pii_count)
        
        return filtered_response

def evaluate_accuracy(response: str, ground_truth: str) -> float:
    """使用 LLM 评估准确性"""
    prompt = f"""
    Compare the following response to the ground truth and rate accuracy from 0 to 1:
    
    Ground Truth: {ground_truth}
    Response: {response}
    
    Score (0-1):
    """
    score = call_llm(prompt)
    return float(score)

# 示例
ground_truth = "Paris is the capital of France."
response = "The capital of France is Paris."
accuracy = evaluate_accuracy(response, ground_truth)  # 1.0

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

model = SentenceTransformer('all-MiniLM-L6-v2')

def evaluate_relevance(query: str, response: str) -> float:
    """计算查询和响应的语义相似度"""
    query_embedding = model.encode([query])
    response_embedding = model.encode([response])
    similarity = cosine_similarity(query_embedding, response_embedding)
    return similarity[0][0]

# 示例
query = "What is the capital of France?"
response = "The capital of France is Paris."
relevance = evaluate_relevance(query, response)  # 0.85

def detect_hallucination(response: str, context: str) -> float:
    """检测响应中是否包含上下文中未提及的信息"""
    prompt = f"""
    Check if the response contains information not supported by the context.
    Rate hallucination from 0 (no hallucination) to 1 (complete hallucination).
    
    Context: {context}
    Response: {response}
    
    Hallucination Score (0-1):
    """
    score = call_llm(prompt)
    return float(score)

# 示例
context = "Paris is the capital of France. It has a population of 2.1 million."
response = "Paris is the capital of France. It was founded in 1500 AD."
hallucination = detect_hallucination(response, context)  # 0.5 (部分幻觉)

from transformers import pipeline

toxicity_classifier = pipeline("text-classification", model="unitary/toxic-bert")

def evaluate_safety(response: str) -> float:
    """检测响应中的毒性内容"""
    result = toxicity_classifier(response)
    toxicity_score = result[0]['score'] if result[0]['label'] == 'toxic' else 0.0
    return 1.0 - toxicity_score  # 安全性 = 1 - 毒性

# 示例
response = "This is a perfectly safe response."
safety = evaluate_safety(response)  # 0.99

# 定时运行 Eval 测试
def run_eval_pipeline():
    # 1. 加载测试数据集
    test_dataset = load_dataset("test_questions.json")
    
    # 2. 运行模型并收集结果
    results = []
    for item in test_dataset:
        response = call_llm(item["question"])
        results.append({
            "question": item["question"],
            "ground_truth": item["answer"],
            "response": response
        })
    
    # 3. 计算 Eval 指标
    metrics = {
        "accuracy": avg([evaluate_accuracy(r["response"], r["ground_truth"]) for r in results]),
        "relevance": avg([evaluate_relevance(r["question"], r["response"]) for r in results]),
        "hallucination": avg([detect_hallucination(r["response"], r["ground_truth"]) for r in results]),
        "safety": avg([evaluate_safety(r["response"]) for r in results])
    }
    
    # 4. 记录到监控系统
    log_metrics(metrics)
    
    # 5. 告警（如果指标下降）
    if metrics["accuracy"] < 0.8:
        alert("Model accuracy dropped below 80%")

# 每天运行一次
schedule.every().day.at("02:00").do(run_eval_pipeline)

from langfuse import Langfuse

langfuse = Langfuse(
    public_key="pk-...",
    secret_key="sk-...",
    host="https://cloud.langfuse.com"
)

def handle_query(query: str, user_id: str):
    # 创建追踪
    trace = langfuse.trace(
        name="rag-query",
        user_id=user_id,
        metadata={"query": query}
    )
    
    # 1. 向量检索
    retrieval_span = trace.span(name="vector-retrieval")
    documents = retrieve_documents(query)
    retrieval_span.end(metadata={"documents_count": len(documents)})
    
    # 2. LLM 调用
    llm_span = trace.span(name="llm-inference")
    response = call_llm(query, documents)
    llm_span.end(
        metadata={
            "model": "gpt-4",
            "tokens_input": response.usage.prompt_tokens,
            "tokens_output": response.usage.completion_tokens
        }
    )
    
    # 3. 记录评估
    trace.score(
        name="relevance",
        value=evaluate_relevance(query, response.content)
    )
    
    return response

# 基准测试集应覆盖：
test_dataset = {
    "simple_questions": [...],      # 简单事实性问题
    "complex_questions": [...],     # 复杂推理问题
    "edge_cases": [...],            # 边界情况
    "adversarial_examples": [...]   # 对抗样本（Prompt 注入等）
}

# 使用多个 LLM 评估，减少偏见
def evaluate_with_multiple_judges(response: str, ground_truth: str):
    judges = ["gpt-4", "claude-3", "llama-3"]
    scores = []
    for judge in judges:
        score = call_llm(judge, f"Rate accuracy: {response} vs {ground_truth}")
        scores.append(score)
    return avg(scores)  # 取平均值

# 自动评估：快速、低成本、覆盖广
# 人工评估：准确、高成本、样本少

# 建议：
# - 自动评估：每天运行，覆盖所有请求
# - 人工评估：每周抽样 100 个案例，验证自动评估的准确性