Skip to content

API Reference

Top Level Module

aspis

Root for the Aspis application core functionality.

api

Module for the Aspis REST API.

main

Functions for the programmatic API of the Aspis application.

EvaluationResponse

Bases: BaseModel

Response for the evaluation endpoint.

Attributes:

Name Type Description
systematized_concept_title str

The title of the systematized concept this evaluation is for.
result dict[str, Any]

The result of the inference of the input text against this systematized concept.
prompt str

The prompt that was used to produce the result.
Source code in src/aspis/api/main.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
class EvaluationResponse(BaseModel):
    """Response for the evaluation endpoint.

    Attributes:
        systematized_concept_title: The title of the systematized concept this
            evaluation is for.
        result: The result of the inference of the input text against this
            systematized concept.
        prompt: The prompt that was used to produce the result.
    """

    systematized_concept_title: str
    result: dict[str, Any]
    prompt: str
evaluate async 
evaluate(
    text_to_evaluate=Form(...),
    api_key=Form(...),
    systematized_concepts_file=File(...),
    model=Form(ModelInfo.OPENAI_GPT_4O),
)

Evaluate an input text using systematized concepts from a file.

Returns one evaluation per systematized concept.

Parameters:

Name Type Description Default
text_to_evaluate str

The text to evaluate.

 Form(...)
api_key str

The API key to use to connect to the model.

 Form(...)
model ModelInfo

The model to use for this evaluation. Optional, defaults to openai/gpt-4o. Allowed values are openai/gpt-4o, openai/gpt-5.5, openai/gpt-5.4-mini, google/gemini-3.1-pro-preview, google/gemini-3-flash-preview, google/gemini-3.1-flash-lite, anthropic/claude-opus-4-7, anthropic/claude-sonnet-4-6 and anthropic/claude-haiku-4-5-20251001.

 Form(OPENAI_GPT_4O)
systematized_concepts_file UploadFile

The file containing the systematized concepts. It must be a .yaml file that contains a systematized_concepts key with a list of systematized concepts. Each systematized concept must contain a title key and a prompt_template key. Example:

systematized_concepts:
- title: "Systematized concept 1"
  prompt_template: "Prompt template 1"
- title: "Systematized concept 2"
  prompt_template: "Prompt template 2"
File(...)

Returns:

Type Description
list[EvaluationResponse]

A list of evaluations for the input text, one for each systematized concept in the file.
Source code in src/aspis/api/main.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@app.post("/evaluate_from_file")
async def evaluate(
    text_to_evaluate: str = Form(...),
    api_key: str = Form(...),
    systematized_concepts_file: UploadFile = File(...),  # noqa: B008 mypy's false positive on File(...)
    model: ModelInfo = Form(ModelInfo.OPENAI_GPT_4O),  # noqa: B008 mypy's false positive on Form with default value
) -> list[EvaluationResponse]:
    """Evaluate an input text using systematized concepts from a file.

    Returns one evaluation per systematized concept.

    Args:
        text_to_evaluate: The text to evaluate.
        api_key: The API key to use to connect to the model.
        model: The model to use for this evaluation. Optional,
            defaults to `openai/gpt-4o`. Allowed values are `openai/gpt-4o`,
            `openai/gpt-5.5`, `openai/gpt-5.4-mini`, `google/gemini-3.1-pro-preview`,
            `google/gemini-3-flash-preview`, `google/gemini-3.1-flash-lite`,
            `anthropic/claude-opus-4-7`, `anthropic/claude-sonnet-4-6`
            and `anthropic/claude-haiku-4-5-20251001`.
        systematized_concepts_file: The file containing the systematized concepts.
            It must be a `.yaml` file that contains a `systematized_concepts` key
            with a list of systematized concepts. Each systematized concept must
            contain a `title` key and a `prompt_template` key. Example:
            \n
                systematized_concepts:
                - title: "Systematized concept 1"
                  prompt_template: "Prompt template 1"
                - title: "Systematized concept 2"
                  prompt_template: "Prompt template 2"

    Returns:
        A list of evaluations for the input text, one for each systematized concept
            in the file.
    """
    try:
        file_content = await systematized_concepts_file.read()
        file_text = file_content.decode("utf-8")

        systematized_concepts_file_content = yaml.safe_load(file_text)

        assert "systematized_concepts" in systematized_concepts_file_content, (
            "The file must contain a 'systematized_concepts' key"
        )

        systematized_concepts = systematized_concepts_file_content["systematized_concepts"]

        prompt_templates = []
        for systematized_concept in systematized_concepts:
            assert "title" in systematized_concept, "Systematized concepts must contain a 'title' key"
            assert "prompt_template" in systematized_concept, (
                "Systematized concepts must contain a 'prompt_template' key"
            )
            prompt_templates.append(systematized_concept["prompt_template"])

        logger.info("%s: Evaluating input text against all concepts...", datetime.datetime.now())

        results = evaluate_text(text_to_evaluate, prompt_templates, model, api_key)

        evaluation_responses = []
        for i in range(len(systematized_concepts)):
            evaluation_responses.append(
                EvaluationResponse(
                    systematized_concept_title=systematized_concepts[i]["title"],
                    result=results[i],
                    prompt=get_inference_prompt(text_to_evaluate, systematized_concepts[i]["prompt_template"]),
                )
            )

        return evaluation_responses

    except AssertionError as e:
        logger.exception("Assertion error during evaluation: %s", e)
        raise HTTPException(status_code=422, detail=str(e)) from e
    except Exception as e:
        logger.exception("Unexpected error during evaluation: %s", e)
        raise HTTPException(status_code=500, detail=f"Evaluation failed: {str(e)}") from e

inferencer

Scorer for applications using Aspis as anLLM-as-a-judge.

ModelInfo

Bases: str, Enum

Information about the supported models for inferencing.

Source code in src/aspis/inferencer.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class ModelInfo(str, Enum):
    """Information about the supported models for inferencing."""

    model_id: str
    friendly_name: str
    api_key_name: str

    OPENAI_GPT_4O = ("openai/gpt-4o", "GPT-4o (OpenAI)", "OPENAI_API_KEY")
    OPENAI_GPT_5_5 = ("openai/gpt-5.5", "GPT-5.5 (OpenAI)", "OPENAI_API_KEY")
    OPENAI_GPT_5_4_MINI = ("openai/gpt-5.4-mini", "GPT-5.4-mini (OpenAI)", "OPENAI_API_KEY")
    GOOGLE_GEMINI_3_1_PRO_PREVIEW = (
        "google/gemini-3.1-pro-preview",
        "Gemini 3.1 Pro Preview (Google)",
        "GOOGLE_API_KEY",
    )
    GOOGLE_GEMINI_3_FLASH_PREVIEW = (
        "google/gemini-3-flash-preview",
        "Gemini 3 Flash Preview (Google)",
        "GOOGLE_API_KEY",
    )
    GOOGLE_GEMINI_3_1_FLASH_LITE = ("google/gemini-3.1-flash-lite", "Gemini 3.1 Flash Lite (Google)", "GOOGLE_API_KEY")
    ANTHROPIC_CLAUDE_4_7_OPUS = ("anthropic/claude-opus-4-7", "Claude Opus 4.7 (Anthropic)", "ANTHROPIC_API_KEY")
    ANTHROPIC_CLAUDE_4_6_SONNET = ("anthropic/claude-sonnet-4-6", "Claude Sonnet 4.6 (Anthropic)", "ANTHROPIC_API_KEY")
    ANTHROPIC_CLAUDE_4_5_HAIKU = (
        "anthropic/claude-haiku-4-5-20251001",
        "Claude Haiku 4.5(Anthropic)",
        "ANTHROPIC_API_KEY",
    )

    def __new__(cls, model_id: str, friendly_name: str, api_key_name: str) -> Self:
        """Make a new ModelInfo enum object.

        The value of the enum will be the model ID.

        Args:
            model_id: The ID of the model.
            friendly_name: The friendly name of the model (displayed in the UI).
            api_key_name: The name of the API key to use to connect to the model.
        """
        obj = str.__new__(cls, model_id)
        obj._value_ = model_id
        obj.model_id = model_id
        obj.friendly_name = friendly_name
        obj.api_key_name = api_key_name
        return obj

    def __str__(self) -> str:
        """Return the friendly name of the model.

        Returns:
            The friendly name of the model.
        """
        return self.friendly_name
__new__ 
__new__(model_id, friendly_name, api_key_name)

Make a new ModelInfo enum object.

The value of the enum will be the model ID.

Parameters:

Name Type Description Default
model_id str

The ID of the model.

 required
friendly_name str

The friendly name of the model (displayed in the UI).

 required
api_key_name str

The name of the API key to use to connect to the model.

 required
Source code in src/aspis/inferencer.py 
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def __new__(cls, model_id: str, friendly_name: str, api_key_name: str) -> Self:
    """Make a new ModelInfo enum object.

    The value of the enum will be the model ID.

    Args:
        model_id: The ID of the model.
        friendly_name: The friendly name of the model (displayed in the UI).
        api_key_name: The name of the API key to use to connect to the model.
    """
    obj = str.__new__(cls, model_id)
    obj._value_ = model_id
    obj.model_id = model_id
    obj.friendly_name = friendly_name
    obj.api_key_name = api_key_name
    return obj
__str__ 
__str__()

Return the friendly name of the model.

Returns:

Type Description
str

The friendly name of the model.
Source code in src/aspis/inferencer.py 
71
72
73
74
75
76
77
def __str__(self) -> str:
    """Return the friendly name of the model.

    Returns:
        The friendly name of the model.
    """
    return self.friendly_name

execute_samples_against_model 

execute_samples_against_model(samples, model_info, api_key)

Executes a list of samples against a model and returns the model outputs.

Parameters:

Name Type Description Default
samples list[Sample]

The list of samples to execute against the model.

 required
model_info ModelInfo

The information about the model to execute the samples against.

 required
api_key str

The API key to use to execute the samples against the model.

 required

Returns:

Type Description
list[str]

The model outputs.
Source code in src/aspis/inferencer.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def execute_samples_against_model(samples: list[Sample], model_info: ModelInfo, api_key: str) -> list[str]:
    """Executes a list of samples against a model and returns the model outputs.

    Args:
        samples: The list of samples to execute against the model.
        model_info: The information about the model to execute the samples against.
        api_key: The API key to use to execute the samples against the model.

    Returns:
        The model outputs.
    """
    logger.info(f"Making API call to model {model_info.model_id}...")

    # Executing this in a synchronous thread pool executor to make InspectAI
    # work well with streamlit's main thread
    with ThreadPoolExecutor() as executor:
        result = executor.submit(run_eval, samples, model_info, api_key).result()

    assert len(result) == 1, "Expected exactly one result"

    if result[0].status != "success":
        logger.error("Evaluation error: %s", result[0].error)
        logger.debug("Full evaluation result: %s", result[0])
        raise ValueError("Error during evaluation.")

    assert result[0].samples is not None, "Expected samples to be not None"
    assert len(result[0].samples) == len(samples), (
        "Expected number of samples to be the same as the number of samples in the task"
    )

    model_outputs = []
    for sample in result[0].samples:
        message_content = extract_string_output(
            sample.output.choices[0].message.content,
            model_info,
        )
        assert isinstance(message_content, str), "Expected message content to be a string"
        model_outputs.append(message_content)

    return model_outputs

evaluate_text 

evaluate_text(
    input_text, prompt_templates, model_info, api_key
)

Evaluates input text using the model and the prompt.

Will use get_inference_prompt function to replace placeholders in the prompt with the input text.

Parameters:

Name Type Description Default
input_text str

The input text to infer.

 required
prompt_templates list[str]

The list of prompt templates to use to infer the input text.

 required
model_info ModelInfo

The information about the model to use to infer the input text.

 required
api_key str

The API key to use to connect to the model.

 required

Returns:

Type Description
list[dict[str, Any]]

The inferred output from the model, parsed from a json to a dictionary.
Source code in src/aspis/inferencer.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def evaluate_text(
    input_text: str, prompt_templates: list[str], model_info: ModelInfo, api_key: str
) -> list[dict[str, Any]]:
    """Evaluates input text using the model and the prompt.

    Will use `get_inference_prompt` function to replace placeholders in the prompt
    with the input text.

    Args:
        input_text: The input text to infer.
        prompt_templates: The list of prompt templates to use to infer the input text.
        model_info: The information about the model to use to infer the input text.
        api_key: The API key to use to connect to the model.

    Returns:
        The inferred output from the model, parsed from a json to a dictionary.
    """
    samples = []
    for prompt_template in prompt_templates:
        input_prompt = get_inference_prompt(input_text, prompt_template)
        samples.append(Sample(input=input_prompt, target=""))

    model_outputs = execute_samples_against_model(samples, model_info, api_key)

    parsed_model_outputs = []
    for model_output in model_outputs:
        cleaned_message_content = clean_model_output(model_output)
        try:
            parsed_message_content = json.loads(cleaned_message_content)
        except Exception:
            logger.exception("Error parsing the model output as json. Writing the raw output to the return.")
            logger.debug("Cleaned message content: %s", cleaned_message_content)
            parsed_message_content = {"raw_output": cleaned_message_content}

        parsed_model_outputs.append(parsed_message_content)

    return parsed_model_outputs

run_eval 

run_eval(samples, model_info, api_key)

Helper function to run eval on a list of samples with a specific API key.

Parameters:

Name Type Description Default
samples list[Sample]

The list of samples to run the eval on.

 required
model_info ModelInfo

The information about the model to use for the evaluation.

 required
api_key str

The API key to use to run the eval.

 required

Returns:

Type Description
list[EvalLog]

The result of the eval.
Source code in src/aspis/inferencer.py 
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
def run_eval(samples: list[Sample], model_info: ModelInfo, api_key: str) -> list[EvalLog]:
    """Helper function to run eval on a list of samples with a specific API key.

    Args:
        samples: The list of samples to run the eval on.
        model_info: The information about the model to use for the evaluation.
        api_key: The API key to use to run the eval.

    Returns:
        The result of the eval.
    """
    task = Task(
        dataset=MemoryDataset(samples),
        solver=[generate()],
        scorer=model_graded_qa(),
    )
    with TemporaryDirectory() as temp_dir, _INSPECTAI_EVAL_LOCK:
        try:
            os.environ[model_info.api_key_name] = api_key
            result = inspect_ai_eval(task, model=model_info.model_id, log_dir=temp_dir)
        finally:
            os.environ.pop(model_info.api_key_name, None)
            # Reset the logger level to the default level since
            # inspectai sets it to WARNING
            logger.setLevel(get_logger_level())

    return result

get_inference_prompt 

get_inference_prompt(input_text, prompt)

Get the inference prompt to be used as input to the model.

It does so by replacing the <text_to_evaluate/> placeholder in the prompt with <text>{input_text}</text>.

Parameters:

Name Type Description Default
input_text str

The input text to infer.

 required
prompt str

The prompt to use to infer the input text.

 required

Returns:

Type Description
str

The inference prompt.
Source code in src/aspis/inferencer.py 
190
191
192
193
194
195
196
197
198
199
200
201
202
203
def get_inference_prompt(input_text: str, prompt: str) -> str:
    """Get the inference prompt to be used as input to the model.

    It does so by replacing the `<text_to_evaluate/>` placeholder in the prompt
    with `<text>{input_text}</text>`.

    Args:
        input_text: The input text to infer.
        prompt: The prompt to use to infer the input text.

    Returns:
        The inference prompt.
    """
    return prompt.replace("<text_to_evaluate/>", f"<text>{input_text}</text>")

extract_string_output 

extract_string_output(model_output, model_info)

Extract the string output from the model output given the model info.

Parameters:

Name Type Description Default
model_output Any

The model output.

 required
model_info ModelInfo

The model info.

 required

Returns:

Type Description
str

The string output.
Source code in src/aspis/inferencer.py 
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def extract_string_output(model_output: Any, model_info: ModelInfo) -> str:
    """Extract the string output from the model output given the model info.

    Args:
        model_output: The model output.
        model_info: The model info.

    Returns:
        The string output.
    """
    if model_info == ModelInfo.OPENAI_GPT_4O:
        return model_output

    if model_info in [
        ModelInfo.OPENAI_GPT_5_5,
        ModelInfo.GOOGLE_GEMINI_3_1_PRO_PREVIEW,
        ModelInfo.GOOGLE_GEMINI_3_FLASH_PREVIEW,
        ModelInfo.GOOGLE_GEMINI_3_1_FLASH_LITE,
    ]:
        # first output is the reasoning, second output is the answer
        return model_output[1].text

    if model_info == ModelInfo.OPENAI_GPT_5_4_MINI:
        return model_output[0].text

    raise ValueError(f"Model info {model_info} not supported")

logging

Logging setup for the Aspis application.

get_logger_level 

get_logger_level()

Get the logging level from the LOG_LEVEL environment variable.

If not set, the default is INFO.

Returns:

Type Description
int

The logging level.
Source code in src/aspis/logging.py 
 7
 8
 9
10
11
12
13
14
15
def get_logger_level() -> int:
    """Get the logging level from the LOG_LEVEL environment variable.

    If not set, the default is INFO.

    Returns:
        The logging level.
    """
    return getattr(logging, os.environ.get("LOG_LEVEL", "INFO").upper(), logging.INFO)

setup_logger 

setup_logger()

Sets up the logging for the given runtime, detected dynamically.

The LOG_LEVEL environment variable is used to set the logging level. If not set, the default is INFO.

Returns:

Type Description
Logger

The configured logger instance.
Source code in src/aspis/logging.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def setup_logger() -> logging.Logger:
    """Sets up the logging for the given runtime, detected dynamically.

    The LOG_LEVEL environment variable is used to set the logging level. If not set,
    the default is INFO.

    Returns:
        The configured logger instance.
    """
    if "uvicorn.access" in logging.Logger.manager.loggerDict:
        logger = logging.getLogger("uvicorn.access")
        logger.setLevel(get_logger_level())
    else:
        logging.basicConfig(
            level=get_logger_level(),
            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
            datefmt="%Y-%m-%d %H:%M:%S",
        )
        logger = logging.getLogger(__name__)

    return logger

systematization

Functions and classes to ask the systematization question.

SystematizedConcept dataclass

A systematized concept with a title, body, and prompt template.

Used for LLM measurement.

Source code in src/aspis/systematization.py 
135
136
137
138
139
140
141
142
143
144
@dataclass
class SystematizedConcept:
    """A systematized concept with a title, body, and prompt template.

    Used for LLM measurement.
    """

    title: str
    body: str
    prompt_template: str

get_systematization_questions 

get_systematization_questions(
    product_description,
    risk_description,
    api_key,
    model_info,
)

Get the systematization questions.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
api_key str

The API key to use the LLM.

 required
model_info ModelInfo

The information about the model to use to generate the systematization questions.

 required

Returns:

Type Description
list[str] | None

The follow up systematization questions. Will be None if the model fails to return a valid JSON.
Source code in src/aspis/systematization.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def get_systematization_questions(
    product_description: str,
    risk_description: str,
    api_key: str,
    model_info: ModelInfo,
) -> list[str] | None:
    """Get the systematization questions.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        api_key: The API key to use the LLM.
        model_info: The information about the model to use to generate the
            systematization questions.

    Returns:
        The follow up systematization questions. Will be None if the model fails to
            return a valid JSON.
    """
    logger.info("Querying model for systematization questions")

    sample = Sample(
        input=SYSTEMATIZATION_PROMPT.format(
            product_description=product_description,
            risk_description=risk_description,
            systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
        ),
        target="",
    )

    model_outputs = execute_samples_against_model([sample], model_info, api_key)
    assert len(model_outputs) == 1, "Expected exactly one model output"
    model_output = model_outputs[0]

    logger.info("Received response from model")
    logger.debug("Model's raw response: %s", model_output)

    cleaned_response = clean_model_output(model_output)
    try:
        parsed_response = json.loads(cleaned_response)
    except Exception as e:
        logger.exception("Error parsing the response from the model: %s. Model response: %s", e, cleaned_response)
        return None

    if not isinstance(parsed_response, list) or not all(isinstance(q, str) for q in parsed_response):
        logger.error("Response is not a list of strings: '%s'", cleaned_response)
        return None

    return parsed_response

get_systematized_concepts 

get_systematized_concepts(
    product_description,
    risk_description,
    questions,
    answers,
    api_key,
    model_info,
)

Generate systematized concepts from the answers to follow-up questions.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
questions list[str]

The follow-up questions that were asked.

 required
answers list[str]

The answers provided by the user.

 required
api_key str

The API key to use the LLM.

 required
model_info ModelInfo

The information about the model to use to generate the systematized concepts.

 required

Returns:

Type Description
list[SystematizedConcept] | None

A list of systematized concepts with titles, bodies, and prompt templates. Will be None if the model fails to return a valid JSON.
Source code in src/aspis/systematization.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
def get_systematized_concepts(
    product_description: str,
    risk_description: str,
    questions: list[str],
    answers: list[str],
    api_key: str,
    model_info: ModelInfo,
) -> list[SystematizedConcept] | None:
    """Generate systematized concepts from the answers to follow-up questions.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        questions: The follow-up questions that were asked.
        answers: The answers provided by the user.
        api_key: The API key to use the LLM.
        model_info: The information about the model to use to generate the
            systematized concepts.

    Returns:
        A list of systematized concepts with titles, bodies, and prompt templates.
            Will be None if the model fails to return a valid JSON.
    """
    # Format questions and answers for the prompt
    logger.info("Querying model for systematized concepts")

    sample = Sample(
        input=get_systematized_concepts_prompt(product_description, risk_description, questions, answers),
        target="",
    )

    model_outputs = execute_samples_against_model([sample], model_info, api_key)
    assert len(model_outputs) == 1, "Expected exactly one model output"
    model_output = model_outputs[0]

    logger.info("Received response from model")
    logger.debug("Model's raw response: %s", model_output)

    cleaned_response = clean_model_output(model_output)
    try:
        concepts_data = json.loads(cleaned_response)
    except Exception as e:
        logger.exception("Error parsing the response from the model: %s. Model response: %s", e, cleaned_response)
        return None

    if not isinstance(concepts_data, list) or not all(isinstance(concept, dict) for concept in concepts_data):
        logger.error("Response is not a list of dictionaries: '%s'", cleaned_response)
        return None

    if not all({"title", "body", "prompt_template"} == set(concept.keys()) for concept in concepts_data):
        logger.error("All concepts must have 'title', 'body', and 'prompt_template' keys: '%s'", cleaned_response)
        return None

    return [SystematizedConcept(**concept) for concept in concepts_data]

get_systematization_questions_prompt 

get_systematization_questions_prompt(
    product_description, risk_description
)

Get the systematization questions prompt.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required

Returns:

Type Description
str

The systematization questions prompt.
Source code in src/aspis/systematization.py 
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
def get_systematization_questions_prompt(product_description: str, risk_description: str) -> str:
    """Get the systematization questions prompt.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.

    Returns:
        The systematization questions prompt.
    """
    return SYSTEMATIZATION_PROMPT.format(
        product_description=product_description,
        risk_description=risk_description,
        systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
    )

get_systematized_concepts_prompt 

get_systematized_concepts_prompt(
    product_description,
    risk_description,
    questions,
    answers,
)

Get the systematized concepts prompt.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
questions list[str]

The follow-up questions that were asked.

 required
answers list[str]

The answers provided by the user.

 required

Returns:

Type Description
str

The systematized concepts prompt.
Source code in src/aspis/systematization.py 
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
def get_systematized_concepts_prompt(
    product_description: str,
    risk_description: str,
    questions: list[str],
    answers: list[str],
) -> str:
    """Get the systematized concepts prompt.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        questions: The follow-up questions that were asked.
        answers: The answers provided by the user.

    Returns:
        The systematized concepts prompt.
    """
    questions_and_answers = "\n".join(
        [f"Q: {question}\nA: {answer}" for question, answer in zip(questions, answers, strict=True)]
    )

    return SYSTEMATIZED_CONCEPTS_PROMPT.format(
        product_description=product_description,
        risk_description=risk_description,
        systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
        questions_and_answers=questions_and_answers,
    )

ui

UI module for the Aspis application.

main

UI for the Aspis application.

main 
main()

Entry point for the Aspis application.

Source code in src/aspis/ui/main.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def main() -> None:
    """Entry point for the Aspis application."""
    # Headers
    st.set_page_config(page_title="Aspis", page_icon="🛡️", layout="centered")
    css_path = Path(__file__).parent.parent / "assets" / "styles.css"
    st.markdown(f"<style>{css_path.read_text()}</style>", unsafe_allow_html=True)
    st.title("🛡️ Aspis")

    # Session state
    api_key = st.session_state.get("api_key", "")
    model_info = st.session_state.get("model_info", ModelInfo.OPENAI_GPT_4O)
    risk_description = st.session_state.get("risk_description", "")
    product_description = st.session_state.get("product_description", "")
    follow_up_questions = st.session_state.get("follow_up_questions", None)
    systematization_answers = st.session_state.get("systematization_answers", None)
    systematized_concepts = st.session_state.get("systematized_concepts", None)

    # Rendering the landing page
    if not model_info or not api_key or not product_description or not risk_description:
        render_landing_page()
        render_upload_button()

    # Generating and rendering the follow up questions
    elif systematization_answers is None:
        # Generate questions if not already generated
        if follow_up_questions is None or len(follow_up_questions) == 0:
            with st.spinner("Generating questions..."):
                follow_up_questions = get_systematization_questions(
                    product_description=product_description,
                    risk_description=risk_description,
                    api_key=api_key,
                    model_info=model_info,
                )

        if follow_up_questions is None or len(follow_up_questions) == 0:
            st.error("Error generating questions. Please try again.")
            return

        st.session_state.follow_up_questions = follow_up_questions

        render_follow_up_questions(follow_up_questions)

    # Generating and rendering the systematized concepts
    else:
        if systematized_concepts is None:
            # Answers have been submitted, generate and display systematized concepts
            with st.spinner("Generating systematized concepts..."):
                systematized_concepts = get_systematized_concepts(
                    product_description=product_description,
                    risk_description=risk_description,
                    questions=follow_up_questions,
                    answers=systematization_answers,
                    api_key=api_key,
                    model_info=model_info,
                )

        if systematized_concepts is None:
            st.error("Error generating systematized concepts. Please try again.")
            return

        st.session_state.systematized_concepts = systematized_concepts

        render_systematized_concepts(systematized_concepts)
render_landing_page 
render_landing_page()

Render the landing page elements.

Source code in src/aspis/ui/main.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
def render_landing_page() -> None:
    """Render the landing page elements."""
    st.markdown("##### Welcome to Aspis!")
    st.markdown(
        "To generate a measurement instrument for an AI risk, please start by answering the following questions:"
    )

    with st.form("input_form"):
        # Product description text area
        current_product_description = st.text_area(
            label="What is the description of your AI-powered product?",
            placeholder="Enter your product description here...",
            help=(
                "Your product description is used to generate a measurement instrument for an AI risk. "
                "Please describe your product in a comprehensive way."
            ),
            key="product_description_input",
        )

        # Risk description text area
        current_risk_description = st.text_area(
            label="What is the AI risk you want to create a measurement instrument for?",
            placeholder="Enter your risk description here...",
            help=(
                "Your risk description is used to generate a risk assessment. Please describe the "
                "AI risk your product is exposed to in order to generate a measurement instrument."
            ),
            key="risk_description_input",
        )

        # Model inputs
        st.markdown(
            '<p style="font-size: 0.875rem; margin-bottom: 0.25rem;">Select the model to use and enter its API key:</p>',
            unsafe_allow_html=True,
        )
        column_model, column_api_key = st.columns([0.3, 0.7])
        with column_model:
            model_options = list(ModelInfo)
            current_model_info = st.selectbox(
                label="Select the model you want to use:",
                label_visibility="collapsed",
                options=model_options,
                index=0,
                key="model_info_input",
            )

        with column_api_key:
            current_api_key = st.text_input(
                label="Enter your API key:",
                label_visibility="collapsed",
                placeholder="Paste your API key here...",
                help="Your API key is used to authenticate your requests to the model API.",
                type="password",
                key="api_key_input",
            )

        if st.form_submit_button("Generate Questions", type="primary", key="generate_questions_button"):
            if current_product_description.strip():
                st.session_state.product_description = current_product_description
            else:
                st.error("Please enter a product description before proceeding.")
                return

            if current_risk_description.strip():
                st.session_state.risk_description = current_risk_description
            else:
                st.error("Please enter a risk description before proceeding.")
                return

            if current_api_key.strip():
                st.session_state.api_key = current_api_key
            else:
                st.error("Please enter an API key before proceeding.")
                return

            if current_model_info is not None:
                st.session_state.model_info = current_model_info
            else:
                st.error("Please select a model before proceeding.")
                return

            # If it gets here, all the inputs are set, so rerun the UI
            st.rerun()
render_follow_up_questions 
render_follow_up_questions(follow_up_questions)

Render the follow up questions to be asked to the user.

Parameters:

Name Type Description Default
follow_up_questions list[str]

The follow up questions.

 required
Source code in src/aspis/ui/main.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def render_follow_up_questions(follow_up_questions: list[str]) -> None:
    """Render the follow up questions to be asked to the user.

    Args:
        follow_up_questions: The follow up questions.
    """
    st.markdown("### Follow Up Questions")

    with st.form("questions_form"):
        current_answers = [""] * len(follow_up_questions)
        for i in range(len(follow_up_questions)):
            current_answers[i] = st.text_area(
                label=rf"{i + 1}\. {follow_up_questions[i]}",
                placeholder="Enter your answer here...",
                key=f"answer_input_{i + 1}",
            )

        if st.form_submit_button("Submit Answers", type="primary", key="submit_answers_button"):
            for i in range(len(current_answers)):
                if not current_answers[i].strip():
                    st.error(f"Please answer question {i + 1}.")
                    return

            st.session_state.systematization_answers = current_answers
            st.rerun()
render_systematized_concepts 
render_systematized_concepts(systematized_concepts)

Render the systematized concepts with titles and bodies.

Parameters:

Name Type Description Default
systematized_concepts list[SystematizedConcept]

The list of systematized concepts to display.

 required
Source code in src/aspis/ui/main.py 
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def render_systematized_concepts(systematized_concepts: list[SystematizedConcept]) -> None:
    """Render the systematized concepts with titles and bodies.

    Args:
        systematized_concepts: The list of systematized concepts to display.
    """
    st.markdown("### Systematized Concepts")

    st.markdown(
        "Based on your answers, the following systematized concepts have been generated. "
        "These represent specific formulations of the background concepts that can be "
        "operationalized into a measurement instrument."
    )

    st.markdown("You can download the results in a YAML file for future use by clicking the button below.")

    render_download_button()

    for i, concept in enumerate(systematized_concepts, 1):
        with st.container():
            st.markdown(f"#### {i}. {concept.title}")
            st.markdown(concept.body)

            with st.expander("📝 Measurement Prompt Template", expanded=False):
                st.markdown("**Use this prompt template with an LLM judge to measure this concept:**")
                st.code(concept.prompt_template, language="text", wrap_lines=True)
                st.markdown("*Replace `<text_to_evaluate/>` with the text you want to evaluate.*")

            if i < len(systematized_concepts):
                st.divider()
render_download_button 
render_download_button()

Render the download button to save the results.

Source code in src/aspis/ui/main.py 
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
def render_download_button() -> None:
    """Render the download button to save the results."""
    file_contents = {
        "product_description": st.session_state.product_description,
        "risk_description": st.session_state.risk_description,
        "follow_up_questions": st.session_state.follow_up_questions,
        "systematization_answers": st.session_state.systematization_answers,
        "systematized_concepts": [asdict(concept) for concept in st.session_state.systematized_concepts],
    }
    yaml_data = yaml.safe_dump(file_contents, default_flow_style=False, allow_unicode=True, sort_keys=False)

    st.download_button(
        label="⬇️ Download results",
        data=yaml_data,
        file_name="systematized_concepts.yaml",
        mime="text/yaml",
    )
render_upload_button 
render_upload_button()

Render the upload button to load saved results.

Source code in src/aspis/ui/main.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def render_upload_button() -> None:
    """Render the upload button to load saved results."""
    st.markdown("##### 🗂️ Upload previously saved results:")
    uploaded_file = st.file_uploader(
        label="*.yaml file",
        type=["yaml", "yml"],
        help="Upload a previously saved YAML file to restore your results.",
        key="upload_file_input",
    )

    if uploaded_file is None:
        return

    # Load and validate the file
    try:
        saved_results = yaml.safe_load(uploaded_file)

        required_keys = [
            "product_description",
            "risk_description",
            "follow_up_questions",
            "systematization_answers",
            "systematized_concepts",
        ]
        for key in required_keys:
            if key not in saved_results:
                raise ValueError(f"Key '{key}' is missing from the saved results.")

        systematized_concepts_required_keys = ["title", "body", "prompt_template"]
        for concept in saved_results["systematized_concepts"]:
            for key in systematized_concepts_required_keys:
                if key not in concept:
                    raise ValueError(f"Key '{key}' is missing from a systematized concept in the saved results.")

    except Exception as e:
        st.error(f"Error loading saved results: {e}")
        return

    st.session_state.product_description = saved_results["product_description"]
    st.session_state.risk_description = saved_results["risk_description"]
    # Note: API key is set to a placeholder because it can't be None,
    # we're restoring saved results and don't need to make new API calls at this stage
    st.session_state.api_key = "placeholder-key"
    st.session_state.follow_up_questions = saved_results["follow_up_questions"]
    st.session_state.systematization_answers = saved_results["systematization_answers"]
    st.session_state.systematized_concepts = [
        SystematizedConcept(**concept) for concept in saved_results["systematized_concepts"]
    ]

    st.rerun()

utils

Utility functions for the Aspis application.

clean_model_output 

clean_model_output(output)

Clean the raw output of the model.

Parameters:

Name Type Description Default
output str

The raw output of the model.

 required

Returns:

Type Description
str

The cleaned output.
Source code in src/aspis/utils.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
def clean_model_output(output: str) -> str:
    """Clean the raw output of the model.

    Args:
        output: The raw output of the model.

    Returns:
        The cleaned output.
    """
    cleaned_output = str(output)
    cleaned_output = cleaned_output.replace("```json", "").replace("```", "")
    return cleaned_output.strip()

UI Module

aspis.ui.main

UI for the Aspis application.

main 

main()

Entry point for the Aspis application.

Source code in src/aspis/ui/main.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def main() -> None:
    """Entry point for the Aspis application."""
    # Headers
    st.set_page_config(page_title="Aspis", page_icon="🛡️", layout="centered")
    css_path = Path(__file__).parent.parent / "assets" / "styles.css"
    st.markdown(f"<style>{css_path.read_text()}</style>", unsafe_allow_html=True)
    st.title("🛡️ Aspis")

    # Session state
    api_key = st.session_state.get("api_key", "")
    model_info = st.session_state.get("model_info", ModelInfo.OPENAI_GPT_4O)
    risk_description = st.session_state.get("risk_description", "")
    product_description = st.session_state.get("product_description", "")
    follow_up_questions = st.session_state.get("follow_up_questions", None)
    systematization_answers = st.session_state.get("systematization_answers", None)
    systematized_concepts = st.session_state.get("systematized_concepts", None)

    # Rendering the landing page
    if not model_info or not api_key or not product_description or not risk_description:
        render_landing_page()
        render_upload_button()

    # Generating and rendering the follow up questions
    elif systematization_answers is None:
        # Generate questions if not already generated
        if follow_up_questions is None or len(follow_up_questions) == 0:
            with st.spinner("Generating questions..."):
                follow_up_questions = get_systematization_questions(
                    product_description=product_description,
                    risk_description=risk_description,
                    api_key=api_key,
                    model_info=model_info,
                )

        if follow_up_questions is None or len(follow_up_questions) == 0:
            st.error("Error generating questions. Please try again.")
            return

        st.session_state.follow_up_questions = follow_up_questions

        render_follow_up_questions(follow_up_questions)

    # Generating and rendering the systematized concepts
    else:
        if systematized_concepts is None:
            # Answers have been submitted, generate and display systematized concepts
            with st.spinner("Generating systematized concepts..."):
                systematized_concepts = get_systematized_concepts(
                    product_description=product_description,
                    risk_description=risk_description,
                    questions=follow_up_questions,
                    answers=systematization_answers,
                    api_key=api_key,
                    model_info=model_info,
                )

        if systematized_concepts is None:
            st.error("Error generating systematized concepts. Please try again.")
            return

        st.session_state.systematized_concepts = systematized_concepts

        render_systematized_concepts(systematized_concepts)

render_landing_page 

render_landing_page()

Render the landing page elements.

Source code in src/aspis/ui/main.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
def render_landing_page() -> None:
    """Render the landing page elements."""
    st.markdown("##### Welcome to Aspis!")
    st.markdown(
        "To generate a measurement instrument for an AI risk, please start by answering the following questions:"
    )

    with st.form("input_form"):
        # Product description text area
        current_product_description = st.text_area(
            label="What is the description of your AI-powered product?",
            placeholder="Enter your product description here...",
            help=(
                "Your product description is used to generate a measurement instrument for an AI risk. "
                "Please describe your product in a comprehensive way."
            ),
            key="product_description_input",
        )

        # Risk description text area
        current_risk_description = st.text_area(
            label="What is the AI risk you want to create a measurement instrument for?",
            placeholder="Enter your risk description here...",
            help=(
                "Your risk description is used to generate a risk assessment. Please describe the "
                "AI risk your product is exposed to in order to generate a measurement instrument."
            ),
            key="risk_description_input",
        )

        # Model inputs
        st.markdown(
            '<p style="font-size: 0.875rem; margin-bottom: 0.25rem;">Select the model to use and enter its API key:</p>',
            unsafe_allow_html=True,
        )
        column_model, column_api_key = st.columns([0.3, 0.7])
        with column_model:
            model_options = list(ModelInfo)
            current_model_info = st.selectbox(
                label="Select the model you want to use:",
                label_visibility="collapsed",
                options=model_options,
                index=0,
                key="model_info_input",
            )

        with column_api_key:
            current_api_key = st.text_input(
                label="Enter your API key:",
                label_visibility="collapsed",
                placeholder="Paste your API key here...",
                help="Your API key is used to authenticate your requests to the model API.",
                type="password",
                key="api_key_input",
            )

        if st.form_submit_button("Generate Questions", type="primary", key="generate_questions_button"):
            if current_product_description.strip():
                st.session_state.product_description = current_product_description
            else:
                st.error("Please enter a product description before proceeding.")
                return

            if current_risk_description.strip():
                st.session_state.risk_description = current_risk_description
            else:
                st.error("Please enter a risk description before proceeding.")
                return

            if current_api_key.strip():
                st.session_state.api_key = current_api_key
            else:
                st.error("Please enter an API key before proceeding.")
                return

            if current_model_info is not None:
                st.session_state.model_info = current_model_info
            else:
                st.error("Please select a model before proceeding.")
                return

            # If it gets here, all the inputs are set, so rerun the UI
            st.rerun()

render_follow_up_questions 

render_follow_up_questions(follow_up_questions)

Render the follow up questions to be asked to the user.

Parameters:

Name Type Description Default
follow_up_questions list[str]

The follow up questions.

 required
Source code in src/aspis/ui/main.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def render_follow_up_questions(follow_up_questions: list[str]) -> None:
    """Render the follow up questions to be asked to the user.

    Args:
        follow_up_questions: The follow up questions.
    """
    st.markdown("### Follow Up Questions")

    with st.form("questions_form"):
        current_answers = [""] * len(follow_up_questions)
        for i in range(len(follow_up_questions)):
            current_answers[i] = st.text_area(
                label=rf"{i + 1}\. {follow_up_questions[i]}",
                placeholder="Enter your answer here...",
                key=f"answer_input_{i + 1}",
            )

        if st.form_submit_button("Submit Answers", type="primary", key="submit_answers_button"):
            for i in range(len(current_answers)):
                if not current_answers[i].strip():
                    st.error(f"Please answer question {i + 1}.")
                    return

            st.session_state.systematization_answers = current_answers
            st.rerun()

render_systematized_concepts 

render_systematized_concepts(systematized_concepts)

Render the systematized concepts with titles and bodies.

Parameters:

Name Type Description Default
systematized_concepts list[SystematizedConcept]

The list of systematized concepts to display.

 required
Source code in src/aspis/ui/main.py 
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def render_systematized_concepts(systematized_concepts: list[SystematizedConcept]) -> None:
    """Render the systematized concepts with titles and bodies.

    Args:
        systematized_concepts: The list of systematized concepts to display.
    """
    st.markdown("### Systematized Concepts")

    st.markdown(
        "Based on your answers, the following systematized concepts have been generated. "
        "These represent specific formulations of the background concepts that can be "
        "operationalized into a measurement instrument."
    )

    st.markdown("You can download the results in a YAML file for future use by clicking the button below.")

    render_download_button()

    for i, concept in enumerate(systematized_concepts, 1):
        with st.container():
            st.markdown(f"#### {i}. {concept.title}")
            st.markdown(concept.body)

            with st.expander("📝 Measurement Prompt Template", expanded=False):
                st.markdown("**Use this prompt template with an LLM judge to measure this concept:**")
                st.code(concept.prompt_template, language="text", wrap_lines=True)
                st.markdown("*Replace `<text_to_evaluate/>` with the text you want to evaluate.*")

            if i < len(systematized_concepts):
                st.divider()

render_download_button 

render_download_button()

Render the download button to save the results.

Source code in src/aspis/ui/main.py 
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
def render_download_button() -> None:
    """Render the download button to save the results."""
    file_contents = {
        "product_description": st.session_state.product_description,
        "risk_description": st.session_state.risk_description,
        "follow_up_questions": st.session_state.follow_up_questions,
        "systematization_answers": st.session_state.systematization_answers,
        "systematized_concepts": [asdict(concept) for concept in st.session_state.systematized_concepts],
    }
    yaml_data = yaml.safe_dump(file_contents, default_flow_style=False, allow_unicode=True, sort_keys=False)

    st.download_button(
        label="⬇️ Download results",
        data=yaml_data,
        file_name="systematized_concepts.yaml",
        mime="text/yaml",
    )

render_upload_button 

render_upload_button()

Render the upload button to load saved results.

Source code in src/aspis/ui/main.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def render_upload_button() -> None:
    """Render the upload button to load saved results."""
    st.markdown("##### 🗂️ Upload previously saved results:")
    uploaded_file = st.file_uploader(
        label="*.yaml file",
        type=["yaml", "yml"],
        help="Upload a previously saved YAML file to restore your results.",
        key="upload_file_input",
    )

    if uploaded_file is None:
        return

    # Load and validate the file
    try:
        saved_results = yaml.safe_load(uploaded_file)

        required_keys = [
            "product_description",
            "risk_description",
            "follow_up_questions",
            "systematization_answers",
            "systematized_concepts",
        ]
        for key in required_keys:
            if key not in saved_results:
                raise ValueError(f"Key '{key}' is missing from the saved results.")

        systematized_concepts_required_keys = ["title", "body", "prompt_template"]
        for concept in saved_results["systematized_concepts"]:
            for key in systematized_concepts_required_keys:
                if key not in concept:
                    raise ValueError(f"Key '{key}' is missing from a systematized concept in the saved results.")

    except Exception as e:
        st.error(f"Error loading saved results: {e}")
        return

    st.session_state.product_description = saved_results["product_description"]
    st.session_state.risk_description = saved_results["risk_description"]
    # Note: API key is set to a placeholder because it can't be None,
    # we're restoring saved results and don't need to make new API calls at this stage
    st.session_state.api_key = "placeholder-key"
    st.session_state.follow_up_questions = saved_results["follow_up_questions"]
    st.session_state.systematization_answers = saved_results["systematization_answers"]
    st.session_state.systematized_concepts = [
        SystematizedConcept(**concept) for concept in saved_results["systematized_concepts"]
    ]

    st.rerun()

Systematization Module

aspis.systematization

Functions and classes to ask the systematization question.

SystematizedConcept dataclass

A systematized concept with a title, body, and prompt template.

Used for LLM measurement.

Source code in src/aspis/systematization.py 
135
136
137
138
139
140
141
142
143
144
@dataclass
class SystematizedConcept:
    """A systematized concept with a title, body, and prompt template.

    Used for LLM measurement.
    """

    title: str
    body: str
    prompt_template: str

get_systematization_questions 

get_systematization_questions(
    product_description,
    risk_description,
    api_key,
    model_info,
)

Get the systematization questions.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
api_key str

The API key to use the LLM.

 required
model_info ModelInfo

The information about the model to use to generate the systematization questions.

 required

Returns:

Type Description
list[str] | None

The follow up systematization questions. Will be None if the model fails to return a valid JSON.
Source code in src/aspis/systematization.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def get_systematization_questions(
    product_description: str,
    risk_description: str,
    api_key: str,
    model_info: ModelInfo,
) -> list[str] | None:
    """Get the systematization questions.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        api_key: The API key to use the LLM.
        model_info: The information about the model to use to generate the
            systematization questions.

    Returns:
        The follow up systematization questions. Will be None if the model fails to
            return a valid JSON.
    """
    logger.info("Querying model for systematization questions")

    sample = Sample(
        input=SYSTEMATIZATION_PROMPT.format(
            product_description=product_description,
            risk_description=risk_description,
            systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
        ),
        target="",
    )

    model_outputs = execute_samples_against_model([sample], model_info, api_key)
    assert len(model_outputs) == 1, "Expected exactly one model output"
    model_output = model_outputs[0]

    logger.info("Received response from model")
    logger.debug("Model's raw response: %s", model_output)

    cleaned_response = clean_model_output(model_output)
    try:
        parsed_response = json.loads(cleaned_response)
    except Exception as e:
        logger.exception("Error parsing the response from the model: %s. Model response: %s", e, cleaned_response)
        return None

    if not isinstance(parsed_response, list) or not all(isinstance(q, str) for q in parsed_response):
        logger.error("Response is not a list of strings: '%s'", cleaned_response)
        return None

    return parsed_response

get_systematized_concepts 

get_systematized_concepts(
    product_description,
    risk_description,
    questions,
    answers,
    api_key,
    model_info,
)

Generate systematized concepts from the answers to follow-up questions.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
questions list[str]

The follow-up questions that were asked.

 required
answers list[str]

The answers provided by the user.

 required
api_key str

The API key to use the LLM.

 required
model_info ModelInfo

The information about the model to use to generate the systematized concepts.

 required

Returns:

Type Description
list[SystematizedConcept] | None

A list of systematized concepts with titles, bodies, and prompt templates. Will be None if the model fails to return a valid JSON.
Source code in src/aspis/systematization.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
def get_systematized_concepts(
    product_description: str,
    risk_description: str,
    questions: list[str],
    answers: list[str],
    api_key: str,
    model_info: ModelInfo,
) -> list[SystematizedConcept] | None:
    """Generate systematized concepts from the answers to follow-up questions.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        questions: The follow-up questions that were asked.
        answers: The answers provided by the user.
        api_key: The API key to use the LLM.
        model_info: The information about the model to use to generate the
            systematized concepts.

    Returns:
        A list of systematized concepts with titles, bodies, and prompt templates.
            Will be None if the model fails to return a valid JSON.
    """
    # Format questions and answers for the prompt
    logger.info("Querying model for systematized concepts")

    sample = Sample(
        input=get_systematized_concepts_prompt(product_description, risk_description, questions, answers),
        target="",
    )

    model_outputs = execute_samples_against_model([sample], model_info, api_key)
    assert len(model_outputs) == 1, "Expected exactly one model output"
    model_output = model_outputs[0]

    logger.info("Received response from model")
    logger.debug("Model's raw response: %s", model_output)

    cleaned_response = clean_model_output(model_output)
    try:
        concepts_data = json.loads(cleaned_response)
    except Exception as e:
        logger.exception("Error parsing the response from the model: %s. Model response: %s", e, cleaned_response)
        return None

    if not isinstance(concepts_data, list) or not all(isinstance(concept, dict) for concept in concepts_data):
        logger.error("Response is not a list of dictionaries: '%s'", cleaned_response)
        return None

    if not all({"title", "body", "prompt_template"} == set(concept.keys()) for concept in concepts_data):
        logger.error("All concepts must have 'title', 'body', and 'prompt_template' keys: '%s'", cleaned_response)
        return None

    return [SystematizedConcept(**concept) for concept in concepts_data]

get_systematization_questions_prompt 

get_systematization_questions_prompt(
    product_description, risk_description
)

Get the systematization questions prompt.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required

Returns:

Type Description
str

The systematization questions prompt.
Source code in src/aspis/systematization.py 
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
def get_systematization_questions_prompt(product_description: str, risk_description: str) -> str:
    """Get the systematization questions prompt.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.

    Returns:
        The systematization questions prompt.
    """
    return SYSTEMATIZATION_PROMPT.format(
        product_description=product_description,
        risk_description=risk_description,
        systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
    )

get_systematized_concepts_prompt 

get_systematized_concepts_prompt(
    product_description,
    risk_description,
    questions,
    answers,
)

Get the systematized concepts prompt.

Parameters:

Name Type Description Default
product_description str

The description of the AI-powered product.

 required
risk_description str

The description of the AI risk the product is exposed to.

 required
questions list[str]

The follow-up questions that were asked.

 required
answers list[str]

The answers provided by the user.

 required

Returns:

Type Description
str

The systematized concepts prompt.
Source code in src/aspis/systematization.py 
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
def get_systematized_concepts_prompt(
    product_description: str,
    risk_description: str,
    questions: list[str],
    answers: list[str],
) -> str:
    """Get the systematized concepts prompt.

    Args:
        product_description: The description of the AI-powered product.
        risk_description: The description of the AI risk the product is exposed to.
        questions: The follow-up questions that were asked.
        answers: The answers provided by the user.

    Returns:
        The systematized concepts prompt.
    """
    questions_and_answers = "\n".join(
        [f"Q: {question}\nA: {answer}" for question, answer in zip(questions, answers, strict=True)]
    )

    return SYSTEMATIZED_CONCEPTS_PROMPT.format(
        product_description=product_description,
        risk_description=risk_description,
        systematization_paper=SYSTEMATIZATION_PAPER_PATH.read_text(),
        questions_and_answers=questions_and_answers,
    )