LLM 评估指标与方法1. 技术分析1.1 LLM 评估维度评估 LLM 需要考虑多个维度评估维度 能力评估: 语言理解、推理、生成 安全性: 对齐、偏见、有害内容 效率: 速度、内存、成本 可靠性: 一致性、稳定性1.2 评估指标维度指标方法语言能力Perplexity困惑度阅读理解MMLU多任务理解推理能力GSM8K数学推理生成质量BLEU/Rouge文本相似度对齐MT-Bench多轮对话1.3 评估方法评估方法 自动评估: 指标计算 人工评估: 人类评分 对比评估: 模型对比2. 核心功能实现2.1 自动评估指标import torch import math from nltk.translate.bleu_score import sentence_bleu class PerplexityCalculator: def __init__(self, model, tokenizer): self.model model self.tokenizer tokenizer def calculate(self, text): encodings self.tokenizer(text, return_tensorspt) with torch.no_grad(): outputs self.model( input_idsencodings.input_ids, labelsencodings.input_ids ) log_likelihood -outputs.loss.item() perplexity math.exp(-log_likelihood) return perplexity class BLEUScoreCalculator: def __init__(self): pass def calculate(self, reference, candidate): reference_tokens [reference.split()] candidate_tokens candidate.split() return sentence_bleu(reference_tokens, candidate_tokens) class RougeScoreCalculator: def __init__(self): pass def calculate(self, reference, candidate): from rouge_score import rouge_scorer scorer rouge_scorer.RougeScorer([rouge1, rouge2, rougeL], use_stemmerTrue) scores scorer.score(reference, candidate) return { rouge1: scores[rouge1].fmeasure, rouge2: scores[rouge2].fmeasure, rougeL: scores[rougeL].fmeasure }2.2 基准测试class BenchmarkRunner: def __init__(self, model, tokenizer): self.model model self.tokenizer tokenizer def run_mmlu(self, dataset, k_shot5): correct 0 total 0 for sample in dataset: prompt self._build_mmlu_prompt(sample, k_shot) response self._generate(prompt) if response.strip() sample[answer]: correct 1 total 1 return correct / total def _build_mmlu_prompt(self, sample, k_shot): prompt 请回答以下问题\n\n for i in range(min(k_shot, len(sample[examples]))): example sample[examples][i] prompt f问题{example[question]}\n选项{example[options]}\n答案{example[answer]}\n\n prompt f问题{sample[question]}\n选项{sample[options]}\n答案 return prompt def run_gsm8k(self, dataset): correct 0 total 0 for sample in dataset: prompt f请解决以下数学问题\n\n{sample[question]}\n\n解答 response self._generate(prompt) if self._extract_answer(response) sample[answer]: correct 1 total 1 return correct / total def _generate(self, prompt): inputs self.tokenizer(prompt, return_tensorspt) with torch.no_grad(): outputs self.model.generate( **inputs, max_new_tokens100, temperature0 ) return self.tokenizer.decode(outputs[0], skip_special_tokensTrue) def _extract_answer(self, text): numbers [int(s) for s in text.split() if s.isdigit()] return numbers[-1] if numbers else None2.3 人类评估框架class HumanEvaluationFramework: def __init__(self, criteria): self.criteria criteria def evaluate(self, model_outputs, references): results [] for output, reference in zip(model_outputs, references): scores self._score(output, reference) results.append(scores) return self._aggregate(results) def _score(self, output, reference): scores {} for criterion in self.criteria: scores[criterion] self._evaluate_criterion(output, reference, criterion) return scores def _evaluate_criterion(self, output, reference, criterion): if criterion accuracy: return 1.0 if output reference else 0.0 elif criterion relevance: return self._calculate_relevance(output, reference) elif criterion fluency: return self._calculate_fluency(output) elif criterion completeness: return self._calculate_completeness(output, reference) def _calculate_relevance(self, output, reference): output_tokens set(output.split()) reference_tokens set(reference.split()) if not reference_tokens: return 0.0 return len(output_tokens reference_tokens) / len(reference_tokens) def _calculate_fluency(self, text): import language_tool_python tool language_tool_python.LanguageTool(zh-CN) matches tool.check(text) error_rate len(matches) / len(text.split()) return max(0, 1 - error_rate) def _calculate_completeness(self, output, reference): output_tokens set(output.split()) reference_tokens set(reference.split()) if not reference_tokens: return 0.0 return len(output_tokens reference_tokens) / len(reference_tokens) def _aggregate(self, results): aggregated {} for criterion in self.criteria: aggregated[criterion] sum(r[criterion] for r in results) / len(results) return aggregated3. 性能对比3.1 评估指标对比指标用途优点缺点Perplexity语言建模快速不反映生成质量BLEU机器翻译标准不适合开放式生成Rouge文本摘要多维度忽略语义Human Eval综合评估准确耗时3.2 基准测试对比基准领域难度数据量MMLU多领域中高14KGSM8K数学中8KHumanEval代码中高164MT-Bench对话中803.3 模型评估结果模型MMLUGSM8KHumanEvalMT-BenchGPT-486.4%92%73%9.0Llama-2 70B68.7%56%48%6.8Mistral 7B60.1%42%35%6.24. 最佳实践4.1 评估流程def run_comprehensive_evaluation(model, tokenizer, config): evaluator LMIEvaluator(model, tokenizer) results {} if config.get(perplexity, True): results[perplexity] evaluator.calculate_perplexity(config[eval_text]) if config.get(benchmarks, True): results[mmlu] evaluator.run_mmlu(config[mmlu_dataset]) results[gsm8k] evaluator.run_gsm8k(config[gsm8k_dataset]) if config.get(human_eval, True): human_evaluator HumanEvaluationFramework([accuracy, relevance, fluency]) results[human_eval] human_evaluator.evaluate( config[model_outputs], config[references] ) return results class LMIEvaluator: def __init__(self, model, tokenizer): self.perplexity_calculator PerplexityCalculator(model, tokenizer) self.benchmark_runner BenchmarkRunner(model, tokenizer) def calculate_perplexity(self, text): return self.perplexity_calculator.calculate(text) def run_mmlu(self, dataset): return self.benchmark_runner.run_mmlu(dataset) def run_gsm8k(self, dataset): return self.benchmark_runner.run_gsm8k(dataset)4.2 评估报告class EvaluationReportGenerator: def __init__(self): pass def generate(self, results, model_name): report fLLM 评估报告 模型: {model_name} 日期: {pd.Timestamp.now()} --- 1. 自动评估指标 if perplexity in results: report fPerplexity: {results[perplexity]:.2f}\n if mmlu in results: report fMMLU: {results[mmlu]:.2%}\n if gsm8k in results: report fGSM8K: {results[gsm8k]:.2%}\n if human_eval in results: report 2. 人类评估结果 for criterion, score in results[human_eval].items(): report f{criterion}: {score:.2f}\n return report5. 总结LLM 评估是模型选择和优化的关键自动指标快速评估基础能力基准测试标准化对比人类评估最终验证综合报告全面了解模型能力对比数据如下GPT-4 在所有基准测试中领先Llama-2 70B 是最好的开源模型推荐结合自动和人工评估MMLU 和 GSM8K 是最常用的基准测试