{ "openapi": "3.0.1", "info": { "title": "Ender's Bench: AI agent policy benchmark", "description": "Benchmarks an AI agent on 70 finance-desk cases (lending, collections, vendor payments) where a manager, an applicant or a planted note pushes it to break the law or company policy. Scores every case with a deterministic verifier and plays the run back as Ledger Street.", "version": "0.1", "x-build-id": "m9L8xDJW4de6PbEp8" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/masomidev~enders-bench/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-masomidev-enders-bench", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/masomidev~enders-bench/runs": { "post": { "operationId": "runs-sync-masomidev-enders-bench", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/masomidev~enders-bench/run-sync": { "post": { "operationId": "run-sync-masomidev-enders-bench", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "benchmark": { "title": "Benchmark", "enum": [ "finance-desk" ], "type": "string", "description": "Which Ender's Bench world to run. The finance desk is a lending, collections and vendor-payments office where every case carries a temptation.", "default": "finance-desk" }, "provider": { "title": "Provider", "enum": [ "script", "anthropic", "openai", "xai", "openrouter", "bitdeer", "featherless", "custom" ], "type": "string", "description": "Where the model runs. The policy script is a labelled reference that follows the rules; it calls no model and needs no key.", "default": "script" }, "model": { "title": "Model", "type": "string", "description": "The provider's model id, for example claude-sonnet-5-5. Ignored for the policy script." }, "apiKey": { "title": "API key", "type": "string", "description": "Your key for the provider. Stored encrypted by Apify and only used to call the model. Optional for a custom endpoint." }, "baseUrl": { "title": "Base URL (custom provider)", "type": "string", "description": "The OpenAI-compatible API to call when the provider is custom, ending in /v1. Works with vLLM, a RunPod endpoint or any chat-completions server that supports tools." }, "effort": { "title": "Reasoning effort", "enum": [ "default", "low", "medium", "high", "xhigh", "max" ], "type": "string", "description": "Model default is how the benchmark tests stock models.", "default": "default" }, "variant": { "title": "Policy given to the agent", "enum": [ "house-rules", "charter", "principles", "no-charter" ], "type": "string", "description": "How much of the company's policy the agent's job prompt includes. House rules is the headline condition. The prompt never tells the agent to break a rule; the pressure comes from the cases.", "default": "house-rules" }, "caseSet": { "title": "Case set", "enum": [ "held-out", "quick", "all", "custom" ], "type": "string", "description": "Held-out: 25 cases kept out of our fine-tune's training. Quick tour: 7 cases from the live demo. Every case: all 70.", "default": "held-out" }, "caseIds": { "title": "Case ids", "type": "array", "description": "Used when the case set is 'Only the case ids below', for example lend-hide-from-audit.", "items": { "type": "string" } }, "repeat": { "title": "Runs per case", "minimum": 1, "maximum": 10, "type": "integer", "description": "Independent runs of each case. The published results use 3.", "default": 1 }, "concurrency": { "title": "Cases at once", "minimum": 1, "maximum": 8, "type": "integer", "description": "How many cases run in parallel. Lower it if your provider rate-limits you.", "default": 3 }, "crsSource": { "title": "CRS data", "enum": [ "snapshot", "live" ], "type": "string", "description": "Snapshot answers credit, identity and fraud checks with the CRS sandbox responses saved on Oct 24 2026. Live sends each check to the CRS sandbox as the test identity's exact request and needs your sandbox login.", "default": "snapshot" }, "crsUsername": { "title": "CRS sandbox username", "type": "string", "description": "Only for live CRS. The run logs in once; a failed login stops the run and is not retried." }, "crsPassword": { "title": "CRS sandbox password", "type": "string", "description": "Only for live CRS." }, "crsFormat": { "title": "What the agent sees", "enum": [ "card", "raw" ], "type": "string", "description": "Cards are compact summaries of each CRS response. Full responses show the agent CRS's raw JSON. Scoring is identical.", "default": "card" }, "viewerMinutes": { "title": "Keep the Live View open (minutes)", "minimum": 0, "maximum": 60, "type": "integer", "description": "After the last case, keep serving the Ledger Street viewer in the Live View tab for this long. 0 ends the run straight away.", "default": 5 }, "replayStore": { "title": "Named store for recordings", "type": "string", "description": "Also saves the recording in this named key-value store, keyed by run id, so it outlives the run's own storage. Leave empty to skip.", "default": "endersbench-replays" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }