Skip to content

Evals

Data structures for fed_rag.evals

BenchmarkExample

Bases: BaseModel

Benchmark example data class.

Source code in src/fed_rag/data_structures/evals.py 
10
11
12
13
14
15
class BenchmarkExample(BaseModel):
    """Benchmark example data class."""

    query: str
    response: str
    context: str | None = None

BenchmarkResult

Bases: BaseModel

Benchmark result data class.

Source code in src/fed_rag/data_structures/evals.py 
18
19
20
21
22
23
24
25
class BenchmarkResult(BaseModel):
    """Benchmark result data class."""

    score: float
    metric_name: str
    num_examples_used: int
    num_total_examples: int
    evaluations_file: str | None

BenchmarkEvaluatedExample

Bases: BaseModel

Evaluated benchmark example data class.

Source code in src/fed_rag/data_structures/evals.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class BenchmarkEvaluatedExample(BaseModel):
    """Evaluated benchmark example data class."""

    score: float
    example: BenchmarkExample
    rag_response: RAGResponse

    def model_dump_json_without_embeddings(self) -> str:
        return self.model_dump_json(
            exclude={
                "rag_response": {
                    "source_nodes": {"__all__": {"node": {"embedding"}}}
                }
            }
        )

AggregationMode

Bases: str, Enum

Mode for aggregating evaluation scores.

Source code in src/fed_rag/data_structures/evals.py 
45
46
47
48
49
50
51
class AggregationMode(str, Enum):
    """Mode for aggregating evaluation scores."""

    AVG = "avg"
    SUM = "sum"
    MAX = "max"
    MIN = "min"