{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://www.anguskit.com/schema/angusmeter/angus-script.json", "title": "AngusScript 2.0", "description": "Schema for AngusMeter 2.0 YAML test scripts (AngusMeter/2.0)", "type": "object", "required": [ "specification", "scenarios" ], "properties": { "specification": { "type": "string", "pattern": "^AngusMeter/\\d+\\.\\d+(\\.\\d+)*$", "description": "Script specification version. Must match 'AngusMeter/X.Y' (e.g. 'AngusMeter/2.0'); optional further numeric segments (e.g. 2.0.1) are allowed." }, "info": { "$ref": "#/definitions/ScriptInfo" }, "environments": { "type": "array", "description": "Named deployment environments (dev, staging, prod, …).", "items": { "$ref": "#/definitions/Environment" } }, "variables": { "type": "array", "description": "Global variables accessible from all scenarios and steps.", "items": { "$ref": "#/definitions/Variable" } }, "configuration": { "$ref": "#/definitions/Configuration" }, "setup": { "type": "array", "description": "Lifecycle hook: steps executed once before the main test run.", "items": { "$ref": "#/definitions/Step" } }, "teardown": { "type": "array", "description": "Lifecycle hook: steps executed once after the main test run.", "items": { "$ref": "#/definitions/Step" } }, "scenarios": { "type": "array", "description": "One or more named test scenarios.", "minItems": 1, "items": { "$ref": "#/definitions/Scenario" } }, "dataSources": { "type": "array", "description": "Top-level data sources referenced by loops or variable expressions.", "items": { "$ref": "#/definitions/DataSource" } } }, "additionalProperties": false, "definitions": { "ScriptInfo": { "type": "object", "description": "Script metadata.", "properties": { "title": { "type": "string", "description": "Human-readable title of the script." }, "description": { "type": "string", "description": "Short description of what the script tests." }, "version": { "type": "string", "description": "Semantic version of this script (e.g. '1.0.0')." }, "author": { "type": "string", "description": "Author or team responsible for this script." }, "tags": { "type": "array", "description": "Tags for categorisation and search.", "items": { "type": "string" } }, "created": { "type": "string", "description": "ISO-8601 creation timestamp." }, "updated": { "type": "string", "description": "ISO-8601 last-updated timestamp." } }, "additionalProperties": false }, "Environment": { "type": "object", "description": "A named deployment environment.", "required": [ "name" ], "properties": { "name": { "type": "string", "description": "Unique identifier (e.g. 'dev', 'staging', 'prod')." }, "baseUrl": { "type": "string", "description": "Base URL for all relative request paths." }, "variables": { "type": "object", "description": "Key/value pairs overriding global variables.", "additionalProperties": { "type": "string" } }, "headers": { "type": "object", "description": "Default HTTP headers for every request.", "additionalProperties": { "type": "string" } } }, "additionalProperties": false }, "Variable": { "type": "object", "description": "A script variable.", "required": [ "name", "value" ], "properties": { "name": { "type": "string", "description": "Variable name used in ${myVar} placeholders." }, "value": { "type": "string", "description": "Initial or constant value." }, "type": { "type": "string", "description": "Value type.", "enum": [ "STRING", "NUMBER", "BOOLEAN", "JSON", "EXPRESSION" ], "default": "STRING" }, "description": { "type": "string", "description": "Human-readable description." } }, "additionalProperties": false }, "Configuration": { "type": "object", "description": "Global execution configuration.", "properties": { "platform": { "type": "string", "description": "Target testing platform.", "enum": [ "HTTP_TESTING", "WEBSOCKET_TESTING", "TCP_TESTING", "JDBC_TESTING", "FTP_TESTING", "SMTP_TESTING", "MQTT_TESTING", "KAFKA_TESTING", "REDIS_TESTING", "RABBITMQ_TESTING", "S3_TESTING", "ELASTICSEARCH_TESTING", "SOAP_TESTING", "LDAP_TESTING", "IMAP_TESTING", "POP3_TESTING", "DUBBO_RPC_TESTING", "CUSTOM_TESTING", "WEB_TESTING", "MOBILE_TESTING", "HYBRID", "LLM_TESTING", "LLM_EVALUATION", "RAG_TESTING", "AGENT_TESTING" ], "default": "HTTP_TESTING" }, "target": { "$ref": "#/definitions/TestTarget" }, "duration": { "$ref": "#/definitions/DurationConfig" }, "concurrency": { "$ref": "#/definitions/ConcurrencyConfig" }, "reportInterval": { "type": "integer", "description": "Intermediate results push interval in seconds.", "default": 5, "minimum": 1 }, "retryPolicy": { "$ref": "#/definitions/RetryPolicy" }, "onError": { "type": "string", "description": "Error handling strategy when a step fails beyond retries.", "enum": [ "CONTINUE", "STOP_THREAD", "STOP_TEST", "START_NEXT_ITERATION" ], "default": "CONTINUE" }, "llm": { "$ref": "#/definitions/LlmConfig" } }, "additionalProperties": false }, "LlmConfig": { "type": "object", "description": "LLM-specific configuration for evaluation and testing.", "properties": { "judge": { "$ref": "#/definitions/JudgeConfiguration", "description": "Default LLM-as-judge for eval metrics; step request.judge overrides. Keys align with runtime judge config map." }, "target": { "$ref": "#/definitions/LlmModelConfig", "description": "Target model being evaluated." }, "contestants": { "type": "array", "description": "Models to compare in arena mode.", "items": { "$ref": "#/definitions/LlmModelConfig" } } }, "additionalProperties": false }, "JudgeConfiguration": { "type": "object", "description": "LLM-as-judge settings injected from configuration.llm.judge (provider, model, temperature, apiKey, baseUrl).", "properties": { "provider": { "type": "string", "description": "Provider id (e.g. openai, anthropic, ollama)." }, "model": { "type": "string" }, "temperature": { "type": "number", "description": "Sampling temperature (e.g. 0–2)." }, "apiKey": { "type": "string" }, "baseUrl": { "type": "string", "description": "Optional API base URL (Azure, proxy, gateway)." } }, "additionalProperties": false }, "LlmModelConfig": { "type": "object", "description": "Configuration for an LLM model provider.", "required": [ "provider", "model" ], "properties": { "name": { "type": "string", "description": "Display name for this model (used in arena reports)." }, "provider": { "type": "string", "description": "Model provider identifier.", "enum": [ "openai", "anthropic", "ollama", "deepseek", "gemini", "custom" ] }, "model": { "type": "string", "description": "Model name (e.g. 'gpt-4o', 'claude-sonnet-4-20250514', 'llama3.1')." }, "apiKey": { "type": "string", "description": "API key (use ${ENV_VAR} for environment variable reference)." }, "baseUrl": { "type": "string", "description": "Custom API endpoint URL (required for ollama or custom providers)." }, "temperature": { "type": "number", "description": "Sampling temperature (0.0–2.0).", "minimum": 0, "maximum": 2, "default": 0 }, "maxTokens": { "type": "integer", "description": "Maximum tokens in the response.", "minimum": 1 } }, "additionalProperties": false }, "TestTarget": { "type": "object", "description": "Whether the test runs for iterations or duration.", "properties": { "type": { "type": "string", "enum": [ "ITERATIONS", "DURATION" ] }, "iterations": { "type": "integer", "description": "Number of full scenario iterations.", "minimum": 1 }, "duration": { "$ref": "#/definitions/DurationConfig" } }, "additionalProperties": false }, "DurationConfig": { "type": "object", "description": "Test duration (value + unit).", "required": [ "value" ], "properties": { "value": { "type": "integer", "description": "Numeric duration value.", "minimum": 1 }, "unit": { "type": "string", "description": "Time unit.", "enum": [ "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS" ], "default": "SECONDS" } }, "additionalProperties": false }, "ConcurrencyConfig": { "type": "object", "description": "Concurrency settings.", "required": [ "threads" ], "properties": { "threads": { "type": "integer", "description": "Number of concurrent virtual users.", "minimum": 1, "default": 1 }, "rampUp": { "type": "integer", "description": "Ramp-up period in seconds.", "minimum": 0, "default": 0 }, "rampDown": { "type": "integer", "description": "Ramp-down period in seconds.", "minimum": 0, "default": 0 } }, "additionalProperties": false }, "RetryPolicy": { "type": "object", "description": "Retry policy for failed steps.", "properties": { "maxRetries": { "type": "integer", "description": "Maximum retry attempts (0 = no retries).", "minimum": 0, "default": 0 }, "delay": { "type": "integer", "description": "Base delay between retries in milliseconds.", "minimum": 0, "default": 1000 }, "backoffMultiplier": { "type": "number", "description": "Exponential backoff multiplier (1.0 = constant).", "minimum": 1.0, "default": 1.0 }, "conditions": { "type": "array", "description": "Failure conditions that trigger retry.", "items": { "type": "string", "enum": [ "ASSERTION_FAILED", "CONNECTION_ERROR", "TIMEOUT" ] } }, "delayUnit": { "type": "string", "enum": [ "MILLISECONDS", "SECONDS" ], "default": "MILLISECONDS" } }, "additionalProperties": false }, "Scenario": { "type": "object", "description": "A named group of steps.", "required": [ "name", "steps" ], "properties": { "name": { "type": "string", "description": "Scenario name used in reports." }, "description": { "type": "string", "description": "Description of what this scenario validates." }, "weight": { "type": "integer", "description": "Relative weight for probabilistic selection.", "minimum": 1, "default": 1 }, "steps": { "type": "array", "description": "Ordered list of steps.", "minItems": 1, "items": { "$ref": "#/definitions/Step" } }, "variables": { "type": "array", "description": "Variables scoped to this scenario.", "items": { "$ref": "#/definitions/Variable" } } }, "additionalProperties": false }, "Step": { "type": "object", "description": "A single executable unit within a scenario.", "required": [ "name" ], "properties": { "name": { "type": "string", "description": "Human-readable name used in reports." }, "description": { "type": "string", "description": "Description of what this step does." }, "plugin": { "type": "string", "description": "Plugin identifier (e.g. 'angus-http', 'angus-playwright')." }, "enabled": { "type": "boolean", "description": "When false, the step is skipped.", "default": true }, "request": { "type": "object", "description": "Plugin-specific request payload.", "additionalProperties": true }, "assertions": { "type": "array", "description": "Assertions evaluated after the step completes.", "items": { "$ref": "#/definitions/Assertion" } }, "extractors": { "type": "array", "description": "Data extractors run after the step completes.", "items": { "$ref": "#/definitions/Extractor" } }, "preProcessors": { "type": "array", "description": "Pre-processor identifiers executed before the request.", "items": { "type": "string" } }, "postProcessors": { "type": "array", "description": "Post-processor identifiers executed after extractors.", "items": { "type": "string" } }, "condition": { "$ref": "#/definitions/Condition" }, "loop": { "$ref": "#/definitions/Loop" }, "retryPolicy": { "$ref": "#/definitions/RetryPolicy" }, "thinkTime": { "$ref": "#/definitions/ThinkTime" } }, "additionalProperties": false }, "Assertion": { "type": "object", "description": "An assertion evaluated after step execution.", "required": [ "type" ], "properties": { "name": { "type": "string", "description": "Label for reporting." }, "type": { "type": "string", "description": "Assertion strategy.", "enum": [ "STATUS_CODE", "RESPONSE_TIME", "BODY_CONTAINS", "BODY_JSON_PATH", "BODY_REGEX", "HEADER_EXISTS", "HEADER_VALUE", "BODY_SIZE", "NOT_NULL", "CUSTOM", "BODY_EQUALS", "BODY_XPATH", "METRIC_SCORE", "TOKEN_COUNT", "API_COST", "LLM_LATENCY", "JSON_SCHEMA", "NO_PII", "RESPONSE_BODY", "URL", "TITLE", "ELEMENT_TEXT", "ELEMENT_VALUE", "ELEMENT_VISIBLE", "ORIENTATION" ] }, "operator": { "type": "string", "description": "Comparison operator applied between actual and expected values.", "enum": [ "EQUALS", "NOT_EQUALS", "CONTAINS", "NOT_CONTAINS", "GREATER_THAN", "LESS_THAN", "GREATER_OR_EQUAL", "LESS_OR_EQUAL", "MATCHES_REGEX" ], "default": "EQUALS" }, "expression": { "type": "string", "description": "Type-specific expression (JSONPath, regex, header name, etc.)." }, "expected": { "type": [ "string", "number", "boolean" ], "description": "Expected value or condition." }, "enabled": { "type": "boolean", "description": "When false, the assertion is skipped.", "default": true } }, "additionalProperties": false }, "Extractor": { "type": "object", "description": "Defines a data-extraction operation.", "required": [ "variable", "type" ], "properties": { "variable": { "type": "string", "description": "Variable name that receives the extracted value." }, "type": { "type": "string", "description": "Extraction strategy.", "enum": [ "JSON_PATH", "XPATH", "REGEX", "HEADER", "COOKIE", "RESPONSE_CODE", "RESPONSE_TIME", "LLM_OUTPUT", "TOKEN_COUNT", "COST", "TOOL_CALLS" ] }, "expression": { "type": "string", "description": "Extraction expression (JSONPath, XPath, regex, header name)." }, "defaultValue": { "type": "string", "description": "Fallback value when expression yields no match." } }, "additionalProperties": false }, "Condition": { "type": "object", "description": "Conditional control-flow block.", "required": [ "expression", "thenSteps" ], "properties": { "expression": { "type": "string", "description": "Boolean expression evaluated at runtime." }, "thenSteps": { "type": "array", "description": "Steps executed when expression is true.", "items": { "$ref": "#/definitions/Step" } }, "elseSteps": { "type": "array", "description": "Steps executed when expression is false.", "items": { "$ref": "#/definitions/Step" } } }, "additionalProperties": false }, "Loop": { "type": "object", "description": "Looping control-flow block.", "required": [ "type" ], "properties": { "type": { "type": "string", "description": "Loop strategy.", "enum": [ "COUNT", "WHILE", "FOR_EACH" ] }, "count": { "type": "integer", "description": "Iteration count for COUNT loops.", "minimum": 1 }, "expression": { "type": "string", "description": "Boolean expression for WHILE loops." }, "variable": { "type": "string", "description": "Variable receiving current row in FOR_EACH loops." }, "dataSource": { "type": "string", "description": "Dataset name or CSV path for FOR_EACH loops." }, "steps": { "type": "array", "description": "Steps executed on each iteration.", "items": { "$ref": "#/definitions/Step" } } }, "additionalProperties": false }, "ThinkTime": { "type": "object", "description": "Configures a think-time pause between steps.", "properties": { "type": { "type": "string", "description": "Think-time distribution strategy.", "enum": [ "FIXED", "RANDOM", "GAUSSIAN" ], "default": "FIXED" }, "value": { "type": "integer", "description": "Fixed delay value (FIXED/GAUSSIAN).", "minimum": 0 }, "min": { "type": "integer", "description": "Minimum delay (RANDOM).", "minimum": 0 }, "max": { "type": "integer", "description": "Maximum delay (RANDOM).", "minimum": 0 }, "unit": { "type": "string", "description": "Time unit.", "enum": [ "MILLISECONDS", "SECONDS" ], "default": "MILLISECONDS" } }, "additionalProperties": false }, "DataSource": { "type": "object", "description": "An external data source referenced by loops or variable expressions.", "required": [ "name", "path" ], "properties": { "name": { "type": "string", "description": "Unique name used to reference this data source." }, "type": { "type": "string", "description": "Data source type.", "enum": [ "CSV", "JSON", "JSONL", "JDBC", "GOLDEN" ], "default": "CSV" }, "columns": { "type": "object", "description": "Column mapping for golden datasets (maps standard fields to source columns).", "properties": { "input": { "type": "string", "description": "Source column for the input/query field." }, "expected_output": { "type": "string", "description": "Source column for the expected/reference answer." }, "context": { "type": "string", "description": "Source column for context documents." }, "retrieval_context": { "type": "string", "description": "Source column for RAG retrieval results." }, "tools_called": { "type": "string", "description": "Source column for actual tool calls." }, "expected_tools": { "type": "string", "description": "Source column for expected tool calls." } }, "additionalProperties": { "type": "string" } }, "path": { "type": "string", "description": "File path or connection string." }, "sharing": { "type": "string", "description": "How data is shared across concurrent virtual users.", "enum": [ "THREAD_LOCAL", "SHARED" ], "default": "THREAD_LOCAL" }, "recycleOnEof": { "type": "boolean", "description": "Restart from beginning when all rows consumed.", "default": true } }, "additionalProperties": false } } }