Introduction: Evaluating LLM outputs is fundamentally different from traditional ML metrics. You can’t just compute accuracy when there’s no single correct answer, and human evaluation doesn’t scale. This guide covers the full spectrum of LLM evaluation: automated metrics like BLEU, ROUGE, and BERTScore for measuring similarity; semantic metrics that capture meaning beyond surface-level matching; LLM-as-judge approaches where models evaluate other models; and frameworks for combining multiple signals into actionable quality scores. Whether you’re evaluating a chatbot, summarization system, or code generator, these techniques will help you measure what matters and catch regressions before they reach production.

Text Similarity Metrics

from dataclasses import dataclass
from typing import Any, Optional
from abc import ABC, abstractmethod
import re
from collections import Counter

@dataclass
class MetricResult:
    """Evaluation metric result."""
    
    name: str
    score: float
    details: dict = None

class TextMetric(ABC):
    """Abstract text similarity metric."""
    
    @abstractmethod
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Compute metric between prediction and reference."""
        pass

class BLEUScore(TextMetric):
    """BLEU score for n-gram overlap."""
    
    def __init__(self, max_n: int = 4):
        self.max_n = max_n
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Compute BLEU score."""
        
        pred_tokens = prediction.lower().split()
        ref_tokens = reference.lower().split()
        
        if not pred_tokens:
            return MetricResult(name="bleu", score=0.0)
        
        # Compute n-gram precisions
        precisions = []
        
        for n in range(1, self.max_n + 1):
            pred_ngrams = self._get_ngrams(pred_tokens, n)
            ref_ngrams = self._get_ngrams(ref_tokens, n)
            
            if not pred_ngrams:
                precisions.append(0.0)
                continue
            
            # Count matches
            matches = sum(
                min(pred_ngrams[ng], ref_ngrams.get(ng, 0))
                for ng in pred_ngrams
            )
            
            precision = matches / sum(pred_ngrams.values())
            precisions.append(precision)
        
        # Geometric mean of precisions
        if 0 in precisions:
            bleu = 0.0
        else:
            import math
            bleu = math.exp(sum(math.log(p) for p in precisions) / len(precisions))
        
        # Brevity penalty
        bp = min(1.0, math.exp(1 - len(ref_tokens) / max(len(pred_tokens), 1)))
        bleu *= bp
        
        return MetricResult(
            name="bleu",
            score=bleu,
            details={"precisions": precisions, "brevity_penalty": bp}
        )
    
    def _get_ngrams(self, tokens: list[str], n: int) -> Counter:
        """Extract n-grams from tokens."""
        
        ngrams = []
        for i in range(len(tokens) - n + 1):
            ngrams.append(tuple(tokens[i:i+n]))
        
        return Counter(ngrams)

class ROUGEScore(TextMetric):
    """ROUGE score for recall-oriented evaluation."""
    
    def __init__(self, rouge_type: str = "rouge-l"):
        self.rouge_type = rouge_type
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Compute ROUGE score."""
        
        pred_tokens = prediction.lower().split()
        ref_tokens = reference.lower().split()
        
        if self.rouge_type == "rouge-1":
            return self._rouge_n(pred_tokens, ref_tokens, 1)
        elif self.rouge_type == "rouge-2":
            return self._rouge_n(pred_tokens, ref_tokens, 2)
        else:  # rouge-l
            return self._rouge_l(pred_tokens, ref_tokens)
    
    def _rouge_n(
        self,
        pred_tokens: list[str],
        ref_tokens: list[str],
        n: int
    ) -> MetricResult:
        """Compute ROUGE-N."""
        
        pred_ngrams = self._get_ngrams(pred_tokens, n)
        ref_ngrams = self._get_ngrams(ref_tokens, n)
        
        if not ref_ngrams:
            return MetricResult(name=f"rouge-{n}", score=0.0)
        
        matches = sum(
            min(pred_ngrams.get(ng, 0), count)
            for ng, count in ref_ngrams.items()
        )
        
        precision = matches / max(sum(pred_ngrams.values()), 1)
        recall = matches / sum(ref_ngrams.values())
        
        f1 = 2 * precision * recall / max(precision + recall, 1e-8)
        
        return MetricResult(
            name=f"rouge-{n}",
            score=f1,
            details={"precision": precision, "recall": recall}
        )
    
    def _rouge_l(
        self,
        pred_tokens: list[str],
        ref_tokens: list[str]
    ) -> MetricResult:
        """Compute ROUGE-L using LCS."""
        
        lcs_length = self._lcs_length(pred_tokens, ref_tokens)
        
        precision = lcs_length / max(len(pred_tokens), 1)
        recall = lcs_length / max(len(ref_tokens), 1)
        
        f1 = 2 * precision * recall / max(precision + recall, 1e-8)
        
        return MetricResult(
            name="rouge-l",
            score=f1,
            details={"lcs_length": lcs_length, "precision": precision, "recall": recall}
        )
    
    def _lcs_length(self, seq1: list, seq2: list) -> int:
        """Compute longest common subsequence length."""
        
        m, n = len(seq1), len(seq2)
        dp = [[0] * (n + 1) for _ in range(m + 1)]
        
        for i in range(1, m + 1):
            for j in range(1, n + 1):
                if seq1[i-1] == seq2[j-1]:
                    dp[i][j] = dp[i-1][j-1] + 1
                else:
                    dp[i][j] = max(dp[i-1][j], dp[i][j-1])
        
        return dp[m][n]
    
    def _get_ngrams(self, tokens: list[str], n: int) -> Counter:
        """Extract n-grams."""
        
        ngrams = []
        for i in range(len(tokens) - n + 1):
            ngrams.append(tuple(tokens[i:i+n]))
        
        return Counter(ngrams)

class ExactMatch(TextMetric):
    """Exact match metric."""
    
    def __init__(self, normalize: bool = True):
        self.normalize = normalize
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Check exact match."""
        
        pred = prediction
        ref = reference
        
        if self.normalize:
            pred = self._normalize(pred)
            ref = self._normalize(ref)
        
        match = pred == ref
        
        return MetricResult(
            name="exact_match",
            score=1.0 if match else 0.0
        )
    
    def _normalize(self, text: str) -> str:
        """Normalize text for comparison."""
        
        text = text.lower().strip()
        text = re.sub(r'\s+', ' ', text)
        text = re.sub(r'[^\w\s]', '', text)
        
        return text

Semantic Similarity Metrics

from dataclasses import dataclass
from typing import Any, Optional
import numpy as np

class BERTScore(TextMetric):
    """BERTScore for semantic similarity."""
    
    def __init__(self, model_name: str = "microsoft/deberta-xlarge-mnli"):
        self.model_name = model_name
        self.model = None
        self.tokenizer = None
    
    def _load_model(self):
        """Lazy load model."""
        
        if self.model is None:
            from transformers import AutoModel, AutoTokenizer
            
            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
            self.model = AutoModel.from_pretrained(self.model_name)
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Compute BERTScore."""
        
        self._load_model()
        
        import torch
        
        # Tokenize
        pred_inputs = self.tokenizer(
            prediction,
            return_tensors="pt",
            truncation=True,
            max_length=512
        )
        
        ref_inputs = self.tokenizer(
            reference,
            return_tensors="pt",
            truncation=True,
            max_length=512
        )
        
        # Get embeddings
        with torch.no_grad():
            pred_outputs = self.model(**pred_inputs)
            ref_outputs = self.model(**ref_inputs)
        
        pred_embeds = pred_outputs.last_hidden_state[0]  # [seq_len, hidden]
        ref_embeds = ref_outputs.last_hidden_state[0]
        
        # Compute cosine similarity matrix
        pred_norm = pred_embeds / pred_embeds.norm(dim=-1, keepdim=True)
        ref_norm = ref_embeds / ref_embeds.norm(dim=-1, keepdim=True)
        
        similarity = torch.mm(pred_norm, ref_norm.t())  # [pred_len, ref_len]
        
        # Precision: max similarity for each prediction token
        precision = similarity.max(dim=1).values.mean().item()
        
        # Recall: max similarity for each reference token
        recall = similarity.max(dim=0).values.mean().item()
        
        # F1
        f1 = 2 * precision * recall / max(precision + recall, 1e-8)
        
        return MetricResult(
            name="bertscore",
            score=f1,
            details={"precision": precision, "recall": recall}
        )

class EmbeddingSimilarity(TextMetric):
    """Embedding-based similarity."""
    
    def __init__(self, model_name: str = "sentence-transformers/all-MiniLM-L6-v2"):
        self.model_name = model_name
        self.model = None
    
    def _load_model(self):
        """Lazy load model."""
        
        if self.model is None:
            from sentence_transformers import SentenceTransformer
            self.model = SentenceTransformer(self.model_name)
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Compute embedding similarity."""
        
        self._load_model()
        
        embeddings = self.model.encode([prediction, reference])
        
        # Cosine similarity
        similarity = np.dot(embeddings[0], embeddings[1]) / (
            np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])
        )
        
        return MetricResult(
            name="embedding_similarity",
            score=float(similarity)
        )

class SemanticEntailment(TextMetric):
    """Check if prediction entails reference."""
    
    def __init__(self, model_name: str = "facebook/bart-large-mnli"):
        self.model_name = model_name
        self.model = None
        self.tokenizer = None
    
    def _load_model(self):
        """Lazy load model."""
        
        if self.model is None:
            from transformers import AutoModelForSequenceClassification, AutoTokenizer
            
            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
            self.model = AutoModelForSequenceClassification.from_pretrained(self.model_name)
    
    def compute(self, prediction: str, reference: str) -> MetricResult:
        """Check entailment."""
        
        self._load_model()
        
        import torch
        
        inputs = self.tokenizer(
            prediction,
            reference,
            return_tensors="pt",
            truncation=True,
            max_length=512
        )
        
        with torch.no_grad():
            outputs = self.model(**inputs)
            probs = torch.softmax(outputs.logits, dim=-1)
        
        # Labels: contradiction, neutral, entailment
        entailment_prob = probs[0][2].item()
        contradiction_prob = probs[0][0].item()
        
        return MetricResult(
            name="entailment",
            score=entailment_prob,
            details={
                "entailment": entailment_prob,
                "contradiction": contradiction_prob,
                "neutral": probs[0][1].item()
            }
        )

LLM-as-Judge Evaluation

from dataclasses import dataclass
from typing import Any, Optional
from abc import ABC, abstractmethod
from enum import Enum
import json

class EvalCriterion(Enum):
    """Evaluation criteria."""
    
    RELEVANCE = "relevance"
    COHERENCE = "coherence"
    FLUENCY = "fluency"
    FACTUALITY = "factuality"
    HELPFULNESS = "helpfulness"
    HARMLESSNESS = "harmlessness"

@dataclass
class JudgeResult:
    """LLM judge result."""
    
    criterion: EvalCriterion
    score: float  # 1-5 scale
    reasoning: str
    confidence: float = 1.0

class LLMJudge(ABC):
    """Abstract LLM judge."""
    
    @abstractmethod
    async def evaluate(
        self,
        prediction: str,
        reference: str = None,
        context: str = None,
        criterion: EvalCriterion = EvalCriterion.RELEVANCE
    ) -> JudgeResult:
        """Evaluate prediction."""
        pass

class SinglePointJudge(LLMJudge):
    """Single-point scoring judge."""
    
    def __init__(self, llm_client: Any):
        self.llm = llm_client
        
        self.prompts = {
            EvalCriterion.RELEVANCE: """Rate the relevance of the response to the query on a scale of 1-5.

Query: {context}
Response: {prediction}

Scoring:
1 - Completely irrelevant
2 - Mostly irrelevant with minor relevant points
3 - Partially relevant
4 - Mostly relevant with minor issues
5 - Highly relevant and on-topic

Provide your rating as JSON: {{"score": <1-5>, "reasoning": ""}}""",

            EvalCriterion.COHERENCE: """Rate the coherence of the response on a scale of 1-5.

Response: {prediction}

Scoring:
1 - Incoherent, impossible to follow
2 - Mostly incoherent with some clear parts
3 - Somewhat coherent but disorganized
4 - Mostly coherent with minor issues
5 - Highly coherent and well-organized

Provide your rating as JSON: {{"score": <1-5>, "reasoning": ""}}""",

            EvalCriterion.FACTUALITY: """Rate the factual accuracy of the response on a scale of 1-5.

Reference: {reference}
Response: {prediction}

Scoring:
1 - Completely inaccurate
2 - Mostly inaccurate with some correct facts
3 - Mix of accurate and inaccurate information
4 - Mostly accurate with minor errors
5 - Completely accurate

Provide your rating as JSON: {{"score": <1-5>, "reasoning": ""}}""",

            EvalCriterion.HELPFULNESS: """Rate how helpful the response is on a scale of 1-5.

Query: {context}
Response: {prediction}

Scoring:
1 - Not helpful at all
2 - Minimally helpful
3 - Somewhat helpful
4 - Very helpful
5 - Extremely helpful and comprehensive

Provide your rating as JSON: {{"score": <1-5>, "reasoning": ""}}"""
        }
    
    async def evaluate(
        self,
        prediction: str,
        reference: str = None,
        context: str = None,
        criterion: EvalCriterion = EvalCriterion.RELEVANCE
    ) -> JudgeResult:
        """Evaluate using single-point scoring."""
        
        prompt_template = self.prompts.get(criterion, self.prompts[EvalCriterion.RELEVANCE])
        
        prompt = prompt_template.format(
            prediction=prediction,
            reference=reference or "",
            context=context or ""
        )
        
        response = await self.llm.complete(prompt)
        
        try:
            result = json.loads(response.content)
            
            return JudgeResult(
                criterion=criterion,
                score=float(result["score"]),
                reasoning=result.get("reasoning", "")
            )
        except Exception:
            return JudgeResult(
                criterion=criterion,
                score=3.0,
                reasoning="Failed to parse response",
                confidence=0.5
            )

class PairwiseJudge(LLMJudge):
    """Pairwise comparison judge."""
    
    def __init__(self, llm_client: Any):
        self.llm = llm_client
    
    async def compare(
        self,
        response_a: str,
        response_b: str,
        context: str,
        criterion: EvalCriterion = EvalCriterion.HELPFULNESS
    ) -> dict:
        """Compare two responses."""
        
        prompt = f"""Compare these two responses and determine which is better for {criterion.value}.

Query: {context}

Response A:
{response_a}

Response B:
{response_b}

Which response is better? Provide your answer as JSON:
{{"winner": "A" or "B" or "tie", "reasoning": "", "confidence": <0-1>}}"""
        
        response = await self.llm.complete(prompt)
        
        try:
            result = json.loads(response.content)
            return result
        except Exception:
            return {"winner": "tie", "reasoning": "Parse error", "confidence": 0.5}
    
    async def evaluate(
        self,
        prediction: str,
        reference: str = None,
        context: str = None,
        criterion: EvalCriterion = EvalCriterion.RELEVANCE
    ) -> JudgeResult:
        """Evaluate by comparing to reference."""
        
        if not reference:
            raise ValueError("Pairwise judge requires reference")
        
        result = await self.compare(prediction, reference, context or "", criterion)
        
        # Convert to score
        if result["winner"] == "A":
            score = 4.0
        elif result["winner"] == "B":
            score = 2.0
        else:
            score = 3.0
        
        return JudgeResult(
            criterion=criterion,
            score=score,
            reasoning=result.get("reasoning", ""),
            confidence=result.get("confidence", 0.8)
        )

class RubricJudge(LLMJudge):
    """Judge using detailed rubric."""
    
    def __init__(self, llm_client: Any, rubric: dict[str, str] = None):
        self.llm = llm_client
        self.rubric = rubric or {}
    
    def set_rubric(self, criterion: EvalCriterion, rubric: str):
        """Set rubric for criterion."""
        self.rubric[criterion.value] = rubric
    
    async def evaluate(
        self,
        prediction: str,
        reference: str = None,
        context: str = None,
        criterion: EvalCriterion = EvalCriterion.RELEVANCE
    ) -> JudgeResult:
        """Evaluate using rubric."""
        
        rubric = self.rubric.get(criterion.value, "Rate from 1-5 based on quality.")
        
        prompt = f"""Evaluate the response using this rubric:

{rubric}

Context: {context or "N/A"}
Reference: {reference or "N/A"}
Response: {prediction}

Provide your evaluation as JSON:
{{"score": <1-5>, "reasoning": ""}}"""
        
        response = await self.llm.complete(prompt)
        
        try:
            result = json.loads(response.content)
            
            return JudgeResult(
                criterion=criterion,
                score=float(result["score"]),
                reasoning=result.get("reasoning", "")
            )
        except Exception:
            return JudgeResult(
                criterion=criterion,
                score=3.0,
                reasoning="Parse error",
                confidence=0.5
            )

Task-Specific Metrics

from dataclasses import dataclass
from typing import Any, Optional
import re

class SummarizationMetrics:
    """Metrics for summarization tasks."""
    
    def __init__(self):
        self.rouge = ROUGEScore("rouge-l")
        self.embedding = EmbeddingSimilarity()
    
    def compute(self, summary: str, source: str, reference: str = None) -> dict:
        """Compute summarization metrics."""
        
        results = {}
        
        # Compression ratio
        results["compression_ratio"] = len(summary) / max(len(source), 1)
        
        # ROUGE against reference
        if reference:
            rouge_result = self.rouge.compute(summary, reference)
            results["rouge_l"] = rouge_result.score
        
        # Semantic similarity to source
        embed_result = self.embedding.compute(summary, source)
        results["source_similarity"] = embed_result.score
        
        # Coverage: what fraction of source sentences are represented
        source_sentences = self._split_sentences(source)
        summary_sentences = self._split_sentences(summary)
        
        covered = 0
        for src_sent in source_sentences:
            for sum_sent in summary_sentences:
                sim = self._sentence_similarity(src_sent, sum_sent)
                if sim > 0.5:
                    covered += 1
                    break
        
        results["coverage"] = covered / max(len(source_sentences), 1)
        
        return results
    
    def _split_sentences(self, text: str) -> list[str]:
        """Split text into sentences."""
        return re.split(r'[.!?]+', text)
    
    def _sentence_similarity(self, sent1: str, sent2: str) -> float:
        """Quick sentence similarity."""
        words1 = set(sent1.lower().split())
        words2 = set(sent2.lower().split())
        
        if not words1 or not words2:
            return 0.0
        
        return len(words1 & words2) / len(words1 | words2)

class QAMetrics:
    """Metrics for question answering."""
    
    def __init__(self):
        self.exact_match = ExactMatch()
    
    def compute(
        self,
        prediction: str,
        reference: str,
        question: str = None
    ) -> dict:
        """Compute QA metrics."""
        
        results = {}
        
        # Exact match
        em_result = self.exact_match.compute(prediction, reference)
        results["exact_match"] = em_result.score
        
        # F1 score (token overlap)
        pred_tokens = set(prediction.lower().split())
        ref_tokens = set(reference.lower().split())
        
        if pred_tokens and ref_tokens:
            precision = len(pred_tokens & ref_tokens) / len(pred_tokens)
            recall = len(pred_tokens & ref_tokens) / len(ref_tokens)
            f1 = 2 * precision * recall / max(precision + recall, 1e-8)
        else:
            f1 = 0.0
        
        results["f1"] = f1
        
        # Contains answer
        results["contains_answer"] = reference.lower() in prediction.lower()
        
        return results

class CodeMetrics:
    """Metrics for code generation."""
    
    def compute(
        self,
        generated_code: str,
        reference_code: str = None,
        test_cases: list[dict] = None
    ) -> dict:
        """Compute code metrics."""
        
        results = {}
        
        # Syntax validity
        results["syntax_valid"] = self._check_syntax(generated_code)
        
        # Code similarity (if reference provided)
        if reference_code:
            results["code_bleu"] = self._code_bleu(generated_code, reference_code)
        
        # Functional correctness (if test cases provided)
        if test_cases:
            passed, total = self._run_tests(generated_code, test_cases)
            results["pass_rate"] = passed / max(total, 1)
            results["tests_passed"] = passed
            results["tests_total"] = total
        
        return results
    
    def _check_syntax(self, code: str) -> bool:
        """Check Python syntax validity."""
        try:
            compile(code, "", "exec")
            return True
        except SyntaxError:
            return False
    
    def _code_bleu(self, generated: str, reference: str) -> float:
        """Compute CodeBLEU-like score."""
        
        # Tokenize code
        gen_tokens = self._tokenize_code(generated)
        ref_tokens = self._tokenize_code(reference)
        
        # N-gram overlap
        bleu = BLEUScore(max_n=4)
        result = bleu.compute(" ".join(gen_tokens), " ".join(ref_tokens))
        
        return result.score
    
    def _tokenize_code(self, code: str) -> list[str]:
        """Tokenize code into meaningful tokens."""
        
        # Simple tokenization
        tokens = re.findall(r'\w+|[^\w\s]', code)
        return tokens
    
    def _run_tests(
        self,
        code: str,
        test_cases: list[dict]
    ) -> tuple[int, int]:
        """Run test cases against code."""
        
        passed = 0
        
        for test in test_cases:
            try:
                # Create isolated namespace
                namespace = {}
                exec(code, namespace)
                
                # Run test
                func_name = test.get("function", "solution")
                func = namespace.get(func_name)
                
                if func:
                    result = func(*test.get("inputs", []))
                    
                    if result == test.get("expected"):
                        passed += 1
            except Exception:
                pass
        
        return passed, len(test_cases)

Evaluation Pipeline

from dataclasses import dataclass, field
from typing import Any, Optional
import asyncio

@dataclass
class EvalSample:
    """Single evaluation sample."""
    
    id: str
    prediction: str
    reference: str = None
    context: str = None
    metadata: dict = field(default_factory=dict)

@dataclass
class EvalReport:
    """Evaluation report."""
    
    metrics: dict[str, float]
    samples: list[dict]
    summary: str = ""

class EvaluationPipeline:
    """Complete evaluation pipeline."""
    
    def __init__(
        self,
        text_metrics: list[TextMetric] = None,
        llm_judge: LLMJudge = None,
        task_metrics: Any = None
    ):
        self.text_metrics = text_metrics or [
            BLEUScore(),
            ROUGEScore("rouge-l"),
            ExactMatch()
        ]
        self.llm_judge = llm_judge
        self.task_metrics = task_metrics
    
    async def evaluate_sample(self, sample: EvalSample) -> dict:
        """Evaluate single sample."""
        
        results = {"id": sample.id}
        
        # Text metrics
        for metric in self.text_metrics:
            if sample.reference:
                result = metric.compute(sample.prediction, sample.reference)
                results[result.name] = result.score
        
        # LLM judge
        if self.llm_judge:
            for criterion in [EvalCriterion.RELEVANCE, EvalCriterion.COHERENCE]:
                judge_result = await self.llm_judge.evaluate(
                    sample.prediction,
                    sample.reference,
                    sample.context,
                    criterion
                )
                results[f"judge_{criterion.value}"] = judge_result.score
        
        # Task-specific metrics
        if self.task_metrics:
            task_results = self.task_metrics.compute(
                sample.prediction,
                sample.reference,
                sample.context
            )
            results.update(task_results)
        
        return results
    
    async def evaluate_batch(
        self,
        samples: list[EvalSample],
        max_concurrent: int = 10
    ) -> EvalReport:
        """Evaluate batch of samples."""
        
        semaphore = asyncio.Semaphore(max_concurrent)
        
        async def eval_with_limit(sample):
            async with semaphore:
                return await self.evaluate_sample(sample)
        
        tasks = [eval_with_limit(s) for s in samples]
        sample_results = await asyncio.gather(*tasks)
        
        # Aggregate metrics
        metric_names = set()
        for result in sample_results:
            metric_names.update(k for k in result.keys() if k != "id")
        
        aggregated = {}
        for name in metric_names:
            values = [r.get(name) for r in sample_results if r.get(name) is not None]
            if values:
                aggregated[name] = sum(values) / len(values)
        
        return EvalReport(
            metrics=aggregated,
            samples=sample_results,
            summary=self._generate_summary(aggregated)
        )
    
    def _generate_summary(self, metrics: dict) -> str:
        """Generate human-readable summary."""
        
        lines = ["Evaluation Summary:", "=" * 40]
        
        for name, value in sorted(metrics.items()):
            lines.append(f"{name}: {value:.4f}")
        
        return "\n".join(lines)

class ABTestEvaluator:
    """Compare two models via A/B testing."""
    
    def __init__(self, judge: PairwiseJudge):
        self.judge = judge
    
    async def compare_models(
        self,
        samples: list[dict],  # {"context": ..., "model_a": ..., "model_b": ...}
        criteria: list[EvalCriterion] = None
    ) -> dict:
        """Compare two models across samples."""
        
        criteria = criteria or [EvalCriterion.HELPFULNESS, EvalCriterion.RELEVANCE]
        
        results = {
            "model_a_wins": 0,
            "model_b_wins": 0,
            "ties": 0,
            "by_criterion": {}
        }
        
        for criterion in criteria:
            results["by_criterion"][criterion.value] = {
                "model_a_wins": 0,
                "model_b_wins": 0,
                "ties": 0
            }
        
        for sample in samples:
            for criterion in criteria:
                comparison = await self.judge.compare(
                    sample["model_a"],
                    sample["model_b"],
                    sample["context"],
                    criterion
                )
                
                winner = comparison.get("winner", "tie")
                
                if winner == "A":
                    results["model_a_wins"] += 1
                    results["by_criterion"][criterion.value]["model_a_wins"] += 1
                elif winner == "B":
                    results["model_b_wins"] += 1
                    results["by_criterion"][criterion.value]["model_b_wins"] += 1
                else:
                    results["ties"] += 1
                    results["by_criterion"][criterion.value]["ties"] += 1
        
        # Calculate win rates
        total = results["model_a_wins"] + results["model_b_wins"] + results["ties"]
        results["model_a_win_rate"] = results["model_a_wins"] / max(total, 1)
        results["model_b_win_rate"] = results["model_b_wins"] / max(total, 1)
        
        return results

Production Evaluation Service

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional

app = FastAPI()

class TextMetricRequest(BaseModel):
    prediction: str
    reference: str
    metrics: list[str] = ["bleu", "rouge-l", "exact_match"]

class JudgeRequest(BaseModel):
    prediction: str
    reference: Optional[str] = None
    context: Optional[str] = None
    criterion: str = "relevance"

class BatchEvalRequest(BaseModel):
    samples: list[dict]
    metrics: list[str] = ["bleu", "rouge-l"]
    use_judge: bool = False

# Initialize metrics
bleu = BLEUScore()
rouge = ROUGEScore("rouge-l")
exact = ExactMatch()

metric_map = {
    "bleu": bleu,
    "rouge-l": rouge,
    "rouge-1": ROUGEScore("rouge-1"),
    "rouge-2": ROUGEScore("rouge-2"),
    "exact_match": exact
}

@app.post("/v1/metrics/text")
async def compute_text_metrics(request: TextMetricRequest) -> dict:
    """Compute text similarity metrics."""
    
    results = {}
    
    for metric_name in request.metrics:
        metric = metric_map.get(metric_name)
        
        if metric:
            result = metric.compute(request.prediction, request.reference)
            results[metric_name] = {
                "score": result.score,
                "details": result.details
            }
    
    return {"metrics": results}

@app.post("/v1/metrics/semantic")
async def compute_semantic_metrics(
    prediction: str,
    reference: str
) -> dict:
    """Compute semantic similarity metrics."""
    
    embedding = EmbeddingSimilarity()
    result = embedding.compute(prediction, reference)
    
    return {
        "embedding_similarity": result.score
    }

@app.post("/v1/judge")
async def llm_judge_evaluate(request: JudgeRequest) -> dict:
    """Evaluate using LLM judge."""
    
    # Note: In production, inject LLM client
    # judge = SinglePointJudge(llm_client)
    
    # Placeholder response
    return {
        "criterion": request.criterion,
        "score": 4.0,
        "reasoning": "Evaluation placeholder - configure LLM client"
    }

@app.post("/v1/evaluate/batch")
async def batch_evaluate(request: BatchEvalRequest) -> dict:
    """Batch evaluation."""
    
    results = []
    
    for sample in request.samples:
        sample_results = {"id": sample.get("id", "unknown")}
        
        prediction = sample.get("prediction", "")
        reference = sample.get("reference", "")
        
        for metric_name in request.metrics:
            metric = metric_map.get(metric_name)
            
            if metric and reference:
                result = metric.compute(prediction, reference)
                sample_results[metric_name] = result.score
        
        results.append(sample_results)
    
    # Aggregate
    aggregated = {}
    for metric_name in request.metrics:
        values = [r.get(metric_name) for r in results if r.get(metric_name) is not None]
        if values:
            aggregated[metric_name] = sum(values) / len(values)
    
    return {
        "samples": results,
        "aggregated": aggregated
    }

@app.get("/health")
async def health():
    return {"status": "healthy"}

References

• BLEU Score: https://aclanthology.org/P02-1040/
• ROUGE: https://aclanthology.org/W04-1013/
• BERTScore: https://arxiv.org/abs/1904.09675
• LLM-as-Judge: https://arxiv.org/abs/2306.05685
• Hugging Face Evaluate: https://huggingface.co/docs/evaluate

Conclusion

LLM evaluation requires a multi-faceted approach because no single metric captures everything that matters. Start with fast, automated metrics like BLEU and ROUGE for quick iteration—they’re not perfect but they catch obvious regressions. Add semantic metrics like BERTScore and embedding similarity to capture meaning beyond surface-level word overlap. For nuanced quality assessment, use LLM-as-judge approaches with clear rubrics and multiple criteria. Pairwise comparison is particularly useful for A/B testing model changes. Task-specific metrics matter: summarization needs coverage and compression ratio, QA needs exact match and F1, code generation needs functional correctness via test execution. Build evaluation into your development workflow—run automated metrics on every change, use LLM judges for periodic deep evaluation, and reserve human evaluation for high-stakes decisions. Track metrics over time to catch gradual degradation. Remember that evaluation is not just about measuring quality—it’s about understanding failure modes and guiding improvement. The best evaluation systems don’t just score outputs; they provide actionable insights about what’s working and what needs attention.