{ "openapi": "3.0.1", "info": { "title": "Actor Benchmark", "description": "Compares various builds of the same actor to measure how they perform on the same input", "version": "0.0", "x-build-id": "RnO8PNdJrueIdyteA" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/apify~actor-benchmark/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-apify-actor-benchmark", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/apify~actor-benchmark/runs": { "post": { "operationId": "runs-sync-apify-actor-benchmark", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/apify~actor-benchmark/run-sync": { "post": { "operationId": "run-sync-apify-actor-benchmark", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "buildNumbers" ], "properties": { "buildNumbers": { "title": "Build numbers or tags", "type": "array", "description": "List of build numbers or tags to benchmark. Each entry will receive an identical run in parallel. You can use exact version numbers (e.g. \"0.3.67\") or tags (e.g. \"canary\"), which will use the latest build associated with that tag.", "items": { "type": "string" } }, "runsPerBuild": { "title": "Runs per build", "minimum": 1, "type": "integer", "description": "Number of times each build is run. Results are averaged to reduce the effect of outliers. All runs are started in parallel on your account, so make sure you have enough memory quota available to allow them all to start simultaneously.", "default": 3 }, "actorId": { "title": "Actor ID", "type": "string", "description": "The ID or name of the Apify actor to benchmark (e.g. \"apify/website-content-crawler\")." }, "actorInput": { "title": "Actor input", "type": "object", "description": "The JSON input passed to every run of the benchmarked actor." }, "taskId": { "title": "Task ID", "type": "string", "description": "The ID or name of an Apify task to benchmark (e.g. \"my-username/my-task\")." }, "maxRunDurationSecs": { "title": "Max run duration (seconds)", "minimum": 10, "type": "integer", "description": "Timeout in seconds for each actor run. Runs exceeding this limit will be aborted." }, "memoryMbytes": { "title": "Memory (MB)", "minimum": 512, "maximum": 32768, "type": "integer", "description": "Memory in megabytes allocated to each actor run." }, "extractRunData": { "title": "Extract function", "type": "string", "description": "A JavaScript function expression called once for every finished run, to pick what you want to measure. It receives one object with: `run` (the run object from the Apify API, refreshed after the run finished), `client` (an authenticated Apify API client, e.g. to read the run's dataset or key-value store), `build` (the build number or tag), `runNumber` (1-based index of the run within its build) and `utils` (`sum`, `mean`, `median` and `round` helpers). It may be async and must return a plain object, e.g. `{ durationSecs: 12.3, emptyItems: 4 }`. The object is stored as is in the \"allRuns\" dataset and passed to \"Aggregate function\". The code runs inside this Actor with your API token; it is not sandboxed. Must be provided together with \"Aggregate function\".", "default": "async ({ run, client, utils }) => {\n const durationSecs = (new Date(run.finishedAt) - new Date(run.startedAt)) / 1000;\n\n let itemCount = 0;\n for await (const _item of client.dataset(run.defaultDatasetId).listItems()) {\n itemCount++;\n }\n\n const totalCost = run.usageTotalUsd ?? 0;\n return {\n durationSecs: utils.round(durationSecs, 2),\n itemCount,\n totalCost: utils.round(totalCost, 6),\n itemsPerSecond: utils.round(durationSecs > 0 ? itemCount / durationSecs : 0, 4),\n costPerItem: utils.round(itemCount > 0 ? totalCost / itemCount : 0, 8),\n };\n}\n" }, "aggregate": { "title": "Aggregate function", "type": "string", "description": "A JavaScript function expression called once for every build, to turn the objects extracted from that build's runs into the statistics to report (averages, percentiles, …). It receives one object with: `build` (the build number or tag), `runs` (array of the objects returned by \"Extract function\" for the build's succeeded or timed-out runs whose extraction worked) and `utils` (`sum`, `mean`, `median` and `round` helpers). It may be async and must return a plain object, e.g. `{ meanDurationSecs: 12.3 }`. The object, plus `build`, becomes one item of the default dataset, and every field that is a number in all builds is plotted in its own chart. Each build is aggregated on its own, so the function cannot compare builds with each other (e.g. against a baseline build).", "default": "({ runs, utils }) => {\n const average = (field, decimals) => utils.round(utils.mean(runs.map((run) => run[field])), decimals);\n\n return {\n durationSecs: average('durationSecs', 2),\n itemCount: average('itemCount', 2),\n totalCost: average('totalCost', 6),\n itemsPerSecond: average('itemsPerSecond', 4),\n costPerItem: average('costPerItem', 8),\n };\n}\n" }, "generateCharts": { "title": "Generate charts", "type": "boolean", "description": "Save one bar chart per numeric field returned by the aggregate function to the key-value store, with the builds on the X axis.", "default": true }, "reportPrompt": { "title": "Report prompt", "type": "string", "description": "Your request to an AI model that reads the statistics returned by the aggregate function and writes a report, e.g. \"Which build is fastest, and is the cost difference worth it?\". Leave empty to not generate a report. Only the statistics of each build, plus which actor or task was benchmarked and how many runs each build had, are sent to the model, not the raw run data. The request goes through the Apify OpenRouter Actor, which starts an additional run billed to your account. The report is saved as \"report.md\" in the key-value store and written to the log. A failed request only logs a warning." }, "reportModel": { "title": "Report model", "type": "string", "description": "OpenRouter model used to write the report.", "default": "anthropic/claude-haiku-4.5" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }