Como Avaliar Modelos de IA

# LLM-as-Judge simplificado
import anthropic

client = anthropic.Anthropic()

rubric = """
Avalie a resposta do modelo nos seguintes criterios (1-5):
1. Corretude factual: a resposta esta correta?
2. Relevancia: responde a pergunta feita?
3. Completude: cobre os pontos importantes?
4. Clareza: e clara e bem organizada?
5. Concisao: e concisa sem ser superficial?

Retorne JSON: {"scores": {"corretude": N, ...}, "total": N, "reasoning": "..."}
"""

def evaluate(question: str, answer: str) -> dict:
    response = client.messages.create(
        model="claude-opus-4-20250514",  # juiz forte
        max_tokens=1024,
        messages=[{
            "role": "user",
            "content": f"""
{rubric}

Pergunta: {question}
Resposta do modelo: {answer}
"""
        }]
    )
    return parse_json(response.content[0].text)

# Rodar em batch
results_a = [evaluate(q, model_a(q)) for q in test_set]
results_b = [evaluate(q, model_b(q)) for q in test_set]
avg_a = sum(r["total"] for r in results_a) / len(results_a)
avg_b = sum(r["total"] for r in results_b) / len(results_b)

# Eval harness mínimo em Python
import asyncio
from anthropic import AsyncAnthropic

client = AsyncAnthropic()

async def eval_one(example: dict, model: str) -> dict:
    resp = await client.messages.create(
        model=model,
        max_tokens=1024,
        messages=[{"role": "user", "content": example["input"]}]
    )
    output = resp.content[0].text
    score = await score_with_judge(output, example["expected"])
    return {"input": example["input"], "output": output,
            "expected": example["expected"], "score": score}

async def run_eval(dataset, model):
    results = await asyncio.gather(
        *[eval_one(ex, model) for ex in dataset]
    )
    avg = sum(r["score"] for r in results) / len(results)
    print(f"{model}: {avg:.2f}/5.0 ({len(results)} exemplos)")
    return results