Suite JSON Schema¶
evalkit schema prints this schema. Point your editor's YAML language server at it to get
completion and validation while you write a suite. For example, with the Red Hat YAML extension
for VS Code, save the output as evalkit.schema.json and add this line to the top of
evals.yaml:
# yaml-language-server: $schema=./evalkit.schema.json
The schema below is generated from the suite models when the site builds.
{
"$defs": {
"AnthropicProviderConfig": {
"additionalProperties": false,
"properties": {
"timeout": {
"default": 60.0,
"description": "Per-request timeout in seconds.",
"exclusiveMinimum": 0,
"title": "Timeout",
"type": "number"
},
"pricing": {
"anyOf": [
{
"$ref": "#/$defs/Pricing"
},
{
"type": "null"
}
],
"default": null
},
"cache": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Override the cache default for this provider (models: on, apps: off).",
"title": "Cache"
},
"provider": {
"const": "anthropic",
"title": "Provider",
"type": "string"
},
"model": {
"title": "Model",
"type": "string"
},
"base_url": {
"default": "https://api.anthropic.com",
"title": "Base Url",
"type": "string"
},
"api_key_env": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": "ANTHROPIC_API_KEY",
"title": "Api Key Env"
},
"anthropic_version": {
"default": "2023-06-01",
"title": "Anthropic Version",
"type": "string"
},
"temperature": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Temperature"
},
"max_tokens": {
"default": 1024,
"exclusiveMinimum": 0,
"title": "Max Tokens",
"type": "integer"
},
"headers": {
"additionalProperties": {
"type": "string"
},
"title": "Headers",
"type": "object"
},
"params": {
"additionalProperties": true,
"title": "Params",
"type": "object"
}
},
"required": [
"provider",
"model"
],
"title": "AnthropicProviderConfig",
"type": "object"
},
"CommandProviderConfig": {
"additionalProperties": false,
"properties": {
"timeout": {
"default": 60.0,
"description": "Per-request timeout in seconds.",
"exclusiveMinimum": 0,
"title": "Timeout",
"type": "number"
},
"pricing": {
"anyOf": [
{
"$ref": "#/$defs/Pricing"
},
{
"type": "null"
}
],
"default": null
},
"cache": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Override the cache default for this provider (models: on, apps: off).",
"title": "Cache"
},
"provider": {
"const": "command",
"title": "Provider",
"type": "string"
},
"command": {
"anyOf": [
{
"type": "string"
},
{
"items": {
"type": "string"
},
"type": "array"
}
],
"description": "Program to run. The prompt goes to stdin unless an argument contains {{prompt}}.",
"title": "Command"
},
"cwd": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Cwd"
},
"env": {
"additionalProperties": {
"type": "string"
},
"title": "Env",
"type": "object"
},
"model": {
"default": "command",
"title": "Model",
"type": "string"
}
},
"required": [
"provider",
"command"
],
"title": "CommandProviderConfig",
"type": "object"
},
"ContainsGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "contains",
"title": "Type",
"type": "string"
},
"value": {
"default": null,
"title": "Value"
},
"mode": {
"default": "all",
"enum": [
"all",
"any"
],
"title": "Mode",
"type": "string"
},
"ignore_case": {
"default": false,
"title": "Ignore Case",
"type": "boolean"
}
},
"required": [
"type"
],
"title": "ContainsGrader",
"type": "object"
},
"ExactGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "exact",
"title": "Type",
"type": "string"
},
"value": {
"default": null,
"description": "Expected output. Defaults to the task's expected.",
"title": "Value"
},
"ignore_case": {
"default": false,
"title": "Ignore Case",
"type": "boolean"
},
"strip": {
"default": true,
"title": "Strip",
"type": "boolean"
}
},
"required": [
"type"
],
"title": "ExactGrader",
"type": "object"
},
"HttpProviderConfig": {
"additionalProperties": false,
"properties": {
"timeout": {
"default": 60.0,
"description": "Per-request timeout in seconds.",
"exclusiveMinimum": 0,
"title": "Timeout",
"type": "number"
},
"pricing": {
"anyOf": [
{
"$ref": "#/$defs/Pricing"
},
{
"type": "null"
}
],
"default": null
},
"cache": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Override the cache default for this provider (models: on, apps: off).",
"title": "Cache"
},
"provider": {
"const": "http",
"title": "Provider",
"type": "string"
},
"url": {
"title": "Url",
"type": "string"
},
"method": {
"default": "POST",
"enum": [
"POST",
"PUT",
"GET"
],
"title": "Method",
"type": "string"
},
"headers": {
"additionalProperties": {
"type": "string"
},
"title": "Headers",
"type": "object"
},
"body": {
"description": "JSON body template. Strings may reference {{prompt}}, {{system}}, {{vars.*}}.",
"title": "Body"
},
"output_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Dotted path into the JSON response, e.g. 'data.answer'.",
"title": "Output Path"
},
"model": {
"default": "http",
"title": "Model",
"type": "string"
}
},
"required": [
"provider",
"url"
],
"title": "HttpProviderConfig",
"type": "object"
},
"JsonSchemaGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "json_schema",
"title": "Type",
"type": "string"
},
"schema": {
"anyOf": [
{
"additionalProperties": true,
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"title": "Schema"
},
"schema_file": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Schema File"
},
"extract": {
"default": true,
"description": "Pull JSON out of Markdown code fences or surrounding prose.",
"title": "Extract",
"type": "boolean"
}
},
"required": [
"type"
],
"title": "JsonSchemaGrader",
"type": "object"
},
"LlmJudgeGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "llm_judge",
"title": "Type",
"type": "string"
},
"rubric": {
"title": "Rubric",
"type": "string"
},
"provider": {
"anyOf": [
{
"discriminator": {
"mapping": {
"anthropic": "#/$defs/AnthropicProviderConfig",
"command": "#/$defs/CommandProviderConfig",
"http": "#/$defs/HttpProviderConfig",
"mock": "#/$defs/MockProviderConfig",
"openai": "#/$defs/OpenAIProviderConfig"
},
"propertyName": "provider"
},
"oneOf": [
{
"$ref": "#/$defs/MockProviderConfig"
},
{
"$ref": "#/$defs/OpenAIProviderConfig"
},
{
"$ref": "#/$defs/AnthropicProviderConfig"
},
{
"$ref": "#/$defs/CommandProviderConfig"
},
{
"$ref": "#/$defs/HttpProviderConfig"
}
]
},
{
"type": "null"
}
],
"default": null,
"description": "Judge model. Defaults to the suite-level 'judge'.",
"title": "Provider"
},
"scale": {
"default": 5,
"description": "Scores range from 1 to scale.",
"maximum": 100,
"minimum": 2,
"title": "Scale",
"type": "integer"
},
"min_score": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"description": "Passing score. Defaults to scale - 1.",
"title": "Min Score"
}
},
"required": [
"type",
"rubric"
],
"title": "LlmJudgeGrader",
"type": "object"
},
"MockProviderConfig": {
"additionalProperties": false,
"properties": {
"timeout": {
"default": 60.0,
"description": "Per-request timeout in seconds.",
"exclusiveMinimum": 0,
"title": "Timeout",
"type": "number"
},
"pricing": {
"anyOf": [
{
"$ref": "#/$defs/Pricing"
},
{
"type": "null"
}
],
"default": null
},
"cache": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Override the cache default for this provider (models: on, apps: off).",
"title": "Cache"
},
"provider": {
"const": "mock",
"title": "Provider",
"type": "string"
},
"model": {
"default": "mock",
"title": "Model",
"type": "string"
},
"responses": {
"items": {
"$ref": "#/$defs/MockRule"
},
"title": "Responses",
"type": "array"
},
"default": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Output when no rule matches. Echoes if unset.",
"title": "Default"
},
"latency_ms": {
"default": 0.0,
"minimum": 0,
"title": "Latency Ms",
"type": "number"
},
"flaky": {
"default": 0.0,
"description": "Probability, per seed and repeat, of returning flaky_output.",
"maximum": 1,
"minimum": 0,
"title": "Flaky",
"type": "number"
},
"flaky_output": {
"default": "",
"title": "Flaky Output",
"type": "string"
},
"fail_times": {
"default": 0,
"description": "Raise a retriable error this many times per request before answering.",
"minimum": 0,
"title": "Fail Times",
"type": "integer"
}
},
"required": [
"provider"
],
"title": "MockProviderConfig",
"type": "object"
},
"MockRule": {
"additionalProperties": false,
"properties": {
"match": {
"description": "Regular expression searched against the rendered prompt.",
"title": "Match",
"type": "string"
},
"output": {
"title": "Output",
"type": "string"
},
"flaky": {
"anyOf": [
{
"maximum": 1,
"minimum": 0,
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"description": "Overrides the provider's flaky.",
"title": "Flaky"
}
},
"required": [
"match",
"output"
],
"title": "MockRule",
"type": "object"
},
"NumericGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "numeric",
"title": "Type",
"type": "string"
},
"value": {
"default": null,
"title": "Value"
},
"abs_tol": {
"default": 0.0,
"minimum": 0,
"title": "Abs Tol",
"type": "number"
},
"rel_tol": {
"default": 0.0,
"minimum": 0,
"title": "Rel Tol",
"type": "number"
},
"pick": {
"default": "last",
"description": "Which number to use when the output contains several.",
"enum": [
"last",
"first"
],
"title": "Pick",
"type": "string"
}
},
"required": [
"type"
],
"title": "NumericGrader",
"type": "object"
},
"OpenAIProviderConfig": {
"additionalProperties": false,
"properties": {
"timeout": {
"default": 60.0,
"description": "Per-request timeout in seconds.",
"exclusiveMinimum": 0,
"title": "Timeout",
"type": "number"
},
"pricing": {
"anyOf": [
{
"$ref": "#/$defs/Pricing"
},
{
"type": "null"
}
],
"default": null
},
"cache": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Override the cache default for this provider (models: on, apps: off).",
"title": "Cache"
},
"provider": {
"const": "openai",
"title": "Provider",
"type": "string"
},
"model": {
"title": "Model",
"type": "string"
},
"base_url": {
"default": "https://api.openai.com/v1",
"title": "Base Url",
"type": "string"
},
"api_key_env": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": "OPENAI_API_KEY",
"title": "Api Key Env"
},
"temperature": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Temperature"
},
"max_tokens": {
"anyOf": [
{
"exclusiveMinimum": 0,
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"title": "Max Tokens"
},
"headers": {
"additionalProperties": {
"type": "string"
},
"title": "Headers",
"type": "object"
},
"params": {
"additionalProperties": true,
"description": "Extra request body fields.",
"title": "Params",
"type": "object"
}
},
"required": [
"provider",
"model"
],
"title": "OpenAIProviderConfig",
"type": "object"
},
"Pricing": {
"additionalProperties": false,
"description": "Prices in USD per million tokens. You supply them; evalkit ships no price table.",
"properties": {
"input_per_mtok": {
"default": 0.0,
"minimum": 0,
"title": "Input Per Mtok",
"type": "number"
},
"output_per_mtok": {
"default": 0.0,
"minimum": 0,
"title": "Output Per Mtok",
"type": "number"
}
},
"title": "Pricing",
"type": "object"
},
"PythonGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "python",
"title": "Type",
"type": "string"
},
"function": {
"description": "'path/to/file.py:func' (relative to the suite) or 'package.module:func'.",
"title": "Function",
"type": "string"
},
"args": {
"additionalProperties": true,
"description": "Keyword arguments to pass.",
"title": "Args",
"type": "object"
}
},
"required": [
"type",
"function"
],
"title": "PythonGrader",
"type": "object"
},
"RegexGrader": {
"additionalProperties": false,
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Label shown in reports. Defaults to the type.",
"title": "Name"
},
"negate": {
"default": false,
"description": "Invert the verdict, e.g. 'must not contain'.",
"title": "Negate",
"type": "boolean"
},
"weight": {
"default": 1.0,
"description": "Weight in the task score.",
"exclusiveMinimum": 0,
"title": "Weight",
"type": "number"
},
"type": {
"const": "regex",
"title": "Type",
"type": "string"
},
"pattern": {
"title": "Pattern",
"type": "string"
},
"ignore_case": {
"default": false,
"title": "Ignore Case",
"type": "boolean"
},
"fullmatch": {
"default": false,
"title": "Fullmatch",
"type": "boolean"
}
},
"required": [
"type",
"pattern"
],
"title": "RegexGrader",
"type": "object"
},
"Settings": {
"additionalProperties": false,
"properties": {
"concurrency": {
"default": 4,
"maximum": 256,
"minimum": 1,
"title": "Concurrency",
"type": "integer"
},
"repeat": {
"default": 1,
"description": "Runs per task, for flakiness stats.",
"maximum": 1000,
"minimum": 1,
"title": "Repeat",
"type": "integer"
},
"retries": {
"default": 2,
"maximum": 20,
"minimum": 0,
"title": "Retries",
"type": "integer"
},
"retry_backoff": {
"default": 0.5,
"description": "Base backoff in seconds (doubles).",
"minimum": 0,
"title": "Retry Backoff",
"type": "number"
},
"seed": {
"default": 0,
"title": "Seed",
"type": "integer"
},
"cache": {
"default": true,
"title": "Cache",
"type": "boolean"
},
"cache_path": {
"default": ".evalkit/cache.sqlite",
"title": "Cache Path",
"type": "string"
},
"fail_under": {
"default": 1.0,
"description": "Minimum share of passing tasks for a zero exit code.",
"maximum": 1,
"minimum": 0,
"title": "Fail Under",
"type": "number"
},
"task_pass_rate": {
"default": 1.0,
"description": "Share of a task's repeats that must pass for the task to pass.",
"maximum": 1,
"minimum": 0,
"title": "Task Pass Rate",
"type": "number"
}
},
"title": "Settings",
"type": "object"
},
"Task": {
"additionalProperties": false,
"properties": {
"id": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Id"
},
"input": {
"default": null,
"title": "Input"
},
"vars": {
"additionalProperties": true,
"title": "Vars",
"type": "object"
},
"expected": {
"default": null,
"title": "Expected"
},
"graders": {
"items": {
"discriminator": {
"mapping": {
"contains": "#/$defs/ContainsGrader",
"exact": "#/$defs/ExactGrader",
"json_schema": "#/$defs/JsonSchemaGrader",
"llm_judge": "#/$defs/LlmJudgeGrader",
"numeric": "#/$defs/NumericGrader",
"python": "#/$defs/PythonGrader",
"regex": "#/$defs/RegexGrader"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/$defs/ExactGrader"
},
{
"$ref": "#/$defs/ContainsGrader"
},
{
"$ref": "#/$defs/RegexGrader"
},
{
"$ref": "#/$defs/JsonSchemaGrader"
},
{
"$ref": "#/$defs/NumericGrader"
},
{
"$ref": "#/$defs/PythonGrader"
},
{
"$ref": "#/$defs/LlmJudgeGrader"
}
]
},
"title": "Graders",
"type": "array"
},
"tags": {
"items": {
"type": "string"
},
"title": "Tags",
"type": "array"
},
"metadata": {
"additionalProperties": true,
"title": "Metadata",
"type": "object"
}
},
"title": "Task",
"type": "object"
}
},
"additionalProperties": false,
"properties": {
"version": {
"const": 1,
"default": 1,
"title": "Version",
"type": "integer"
},
"name": {
"title": "Name",
"type": "string"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Description"
},
"target": {
"discriminator": {
"mapping": {
"anthropic": "#/$defs/AnthropicProviderConfig",
"command": "#/$defs/CommandProviderConfig",
"http": "#/$defs/HttpProviderConfig",
"mock": "#/$defs/MockProviderConfig",
"openai": "#/$defs/OpenAIProviderConfig"
},
"propertyName": "provider"
},
"oneOf": [
{
"$ref": "#/$defs/MockProviderConfig"
},
{
"$ref": "#/$defs/OpenAIProviderConfig"
},
{
"$ref": "#/$defs/AnthropicProviderConfig"
},
{
"$ref": "#/$defs/CommandProviderConfig"
},
{
"$ref": "#/$defs/HttpProviderConfig"
}
],
"title": "Target"
},
"judge": {
"anyOf": [
{
"discriminator": {
"mapping": {
"anthropic": "#/$defs/AnthropicProviderConfig",
"command": "#/$defs/CommandProviderConfig",
"http": "#/$defs/HttpProviderConfig",
"mock": "#/$defs/MockProviderConfig",
"openai": "#/$defs/OpenAIProviderConfig"
},
"propertyName": "provider"
},
"oneOf": [
{
"$ref": "#/$defs/MockProviderConfig"
},
{
"$ref": "#/$defs/OpenAIProviderConfig"
},
{
"$ref": "#/$defs/AnthropicProviderConfig"
},
{
"$ref": "#/$defs/CommandProviderConfig"
},
{
"$ref": "#/$defs/HttpProviderConfig"
}
]
},
{
"type": "null"
}
],
"default": null,
"title": "Judge"
},
"system": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "System"
},
"prompt": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Prompt template. Tasks fill it through input, vars, and expected.",
"title": "Prompt"
},
"graders": {
"description": "Graders applied to every task.",
"items": {
"discriminator": {
"mapping": {
"contains": "#/$defs/ContainsGrader",
"exact": "#/$defs/ExactGrader",
"json_schema": "#/$defs/JsonSchemaGrader",
"llm_judge": "#/$defs/LlmJudgeGrader",
"numeric": "#/$defs/NumericGrader",
"python": "#/$defs/PythonGrader",
"regex": "#/$defs/RegexGrader"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/$defs/ExactGrader"
},
{
"$ref": "#/$defs/ContainsGrader"
},
{
"$ref": "#/$defs/RegexGrader"
},
{
"$ref": "#/$defs/JsonSchemaGrader"
},
{
"$ref": "#/$defs/NumericGrader"
},
{
"$ref": "#/$defs/PythonGrader"
},
{
"$ref": "#/$defs/LlmJudgeGrader"
}
]
},
"title": "Graders",
"type": "array"
},
"tasks": {
"items": {
"$ref": "#/$defs/Task"
},
"minItems": 1,
"title": "Tasks",
"type": "array"
},
"settings": {
"$ref": "#/$defs/Settings"
}
},
"required": [
"name",
"target",
"tasks"
],
"title": "evalkit suite",
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema"
}