Index

A

answer (chatbot_eval.types.BotResult attribute) answer() (chatbot_eval.bots.base.BaseBot method) (chatbot_eval.bots.full_context.FullContextBot method) (chatbot_eval.bots.semantic_match.StrictSemanticMatchBot method)

AnswerLengthMetric (class in chatbot_eval.metrics.basic)

B

BaseBot (class in chatbot_eval.bots.base) BotFactory (class in chatbot_eval.bots.factory) BotResult (class in chatbot_eval.types)

build_bot_from_config() (in module chatbot_eval.config.runtime) build_chat_client() (in module chatbot_eval.config.builders) build_default_metrics() (in module chatbot_eval.metrics.registry) build_judge_metric() (in module chatbot_eval.config.builders)

C

chat_client (chatbot_eval.bots.full_context.FullContextBot attribute) chatbot_eval.bots.base module chatbot_eval.bots.factory module chatbot_eval.bots.full_context module chatbot_eval.bots.semantic_match module chatbot_eval.config.builders module chatbot_eval.config.judges module chatbot_eval.config.models module chatbot_eval.config.runtime module

chatbot_eval.evaluation.evaluator module chatbot_eval.io.csv_loader module chatbot_eval.io.reporting module chatbot_eval.metrics.basic module chatbot_eval.metrics.llm_judge module chatbot_eval.metrics.registry module chatbot_eval.types module chatbot_eval.utils.files module Completion (class in chatbot_eval.types) credentials (chatbot_eval.config.models.ModelConfig attribute)

D

debug (chatbot_eval.config.judges.JudgeConfig attribute) (chatbot_eval.metrics.llm_judge.LLMJudgeMetric attribute)

details (chatbot_eval.types.MetricResult attribute) domain_knowledge_path (chatbot_eval.bots.full_context.FullContextBot attribute)

E

evaluate_dataset() (chatbot_eval.evaluation.evaluator.Evaluator method) evaluate_sample() (chatbot_eval.evaluation.evaluator.Evaluator method)

Evaluator (class in chatbot_eval.evaluation.evaluator) ExactMatchMetric (class in chatbot_eval.metrics.basic) expected_answer (chatbot_eval.types.Sample attribute)

F

faq_csv_path (chatbot_eval.bots.semantic_match.StrictSemanticMatchBot attribute)

from_config() (chatbot_eval.bots.factory.BotFactory static method) FullContextBot (class in chatbot_eval.bots.full_context)

J

JudgeConfig (class in chatbot_eval.config.judges)

K

KeywordRecallMetric (class in chatbot_eval.metrics.basic)

L

llm_client (chatbot_eval.metrics.llm_judge.LLMJudgeMetric attribute) LLMJudgeMetric (class in chatbot_eval.metrics.llm_judge) load_json() (in module chatbot_eval.utils.files)

load_judge_config() (in module chatbot_eval.config.builders) load_model_config() (in module chatbot_eval.config.builders) load_samples_from_csv() (in module chatbot_eval.io.csv_loader) load_text() (in module chatbot_eval.utils.files)

M

metadata (chatbot_eval.types.BotResult attribute) MetricResult (class in chatbot_eval.types) metrics (chatbot_eval.evaluation.evaluator.Evaluator attribute) model (chatbot_eval.config.models.ModelConfig attribute) model_config (chatbot_eval.config.judges.JudgeConfig attribute) ModelConfig (class in chatbot_eval.config.models) module chatbot_eval.bots.base chatbot_eval.bots.factory chatbot_eval.bots.full_context chatbot_eval.bots.semantic_match chatbot_eval.config.builders chatbot_eval.config.judges chatbot_eval.config.models chatbot_eval.config.runtime chatbot_eval.evaluation.evaluator chatbot_eval.io.csv_loader chatbot_eval.io.reporting chatbot_eval.metrics.basic chatbot_eval.metrics.llm_judge chatbot_eval.metrics.registry chatbot_eval.types chatbot_eval.utils.files

N

name (chatbot_eval.bots.base.BaseBot attribute) (chatbot_eval.bots.full_context.FullContextBot attribute) (chatbot_eval.bots.semantic_match.StrictSemanticMatchBot attribute) (chatbot_eval.config.judges.JudgeConfig attribute) (chatbot_eval.config.models.ModelConfig attribute) (chatbot_eval.metrics.basic.AnswerLengthMetric attribute) (chatbot_eval.metrics.basic.ExactMatchMetric attribute) (chatbot_eval.metrics.basic.KeywordRecallMetric attribute) (chatbot_eval.metrics.basic.PolitenessMetric attribute) (chatbot_eval.metrics.llm_judge.LLMJudgeMetric attribute) (chatbot_eval.types.MetricResult attribute)

P

PolitenessMetric (class in chatbot_eval.metrics.basic) prompt_path (chatbot_eval.bots.full_context.FullContextBot attribute) (chatbot_eval.config.judges.JudgeConfig attribute) (chatbot_eval.metrics.llm_judge.LLMJudgeMetric attribute)

provider (chatbot_eval.config.models.ModelConfig attribute)

Q

question (chatbot_eval.types.Sample attribute)

R

raw (chatbot_eval.types.Completion attribute), [1] render_template() (in module chatbot_eval.utils.files)

request_kwargs (chatbot_eval.config.models.ModelConfig attribute) resolve_env() (in module chatbot_eval.utils.files)

S

Sample (class in chatbot_eval.types) score (chatbot_eval.types.MetricResult attribute) score() (chatbot_eval.metrics.basic.AnswerLengthMetric method) (chatbot_eval.metrics.basic.ExactMatchMetric method) (chatbot_eval.metrics.basic.KeywordRecallMetric method) (chatbot_eval.metrics.basic.PolitenessMetric method) (chatbot_eval.metrics.llm_judge.LLMJudgeMetric method)

StrictSemanticMatchBot (class in chatbot_eval.bots.semantic_match) summarize_by_bot() (in module chatbot_eval.io.reporting)

T

temperature (chatbot_eval.config.models.ModelConfig attribute)

text (chatbot_eval.types.Completion attribute), [1] thinking (chatbot_eval.types.Completion attribute), [1]

W

write_rows_csv() (in module chatbot_eval.io.reporting)

write_rows_jsonl() (in module chatbot_eval.io.reporting)