{ "openapi": "3.0.1", "info": { "title": "Crawlability & AI Crawler Access Auditor", "description": "AI crawler access audit and robots.txt checker for any domain: rules per crawler (GPTBot, ClaudeBot, CCBot, Google-Extended and 25 more), Content-Signal and Content-Usage, sitemaps with URL counts, llms.txt, meta robots on sample pages, TDM opt-out detection and the anti-bot service in front.", "version": "0.1", "x-build-id": "ok30XfOKBF8MjFXfV" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/lwsdjfls~crawlability-ai-access-auditor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-lwsdjfls-crawlability-ai-access-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/lwsdjfls~crawlability-ai-access-auditor/runs": { "post": { "operationId": "runs-sync-lwsdjfls-crawlability-ai-access-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/lwsdjfls~crawlability-ai-access-auditor/run-sync": { "post": { "operationId": "run-sync-lwsdjfls-crawlability-ai-access-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "domains" ], "properties": { "domains": { "title": "Domains", "minItems": 1, "maxItems": 1000, "type": "array", "description": "One per line: example.com or https://www.example.com/. Up to 1,000.", "items": { "type": "string", "minLength": 3, "maxLength": 300 } }, "mode": { "title": "Mode", "enum": [ "audit", "monitor" ], "type": "string", "description": "audit: deliver the full audit of every domain. monitor: for a saved task on a schedule, deliver the first audit as a baseline and afterwards only domains whose policy changed (new blocks on AI crawlers, a TDM reservation, an anti-bot service, llms.txt appearing).", "default": "audit" }, "samplePages": { "title": "Pages to sample", "minimum": 0, "maximum": 50, "type": "integer", "description": "Pages fetched per domain for meta robots, X-Robots-Tag and TDM headers: the homepage plus URLs spread over the sitemaps (or homepage links). 0 checks only the homepage.", "default": 8 }, "maxSitemaps": { "title": "Maximum sitemap files", "minimum": 0, "maximum": 500, "type": "integer", "description": "Sitemap index and child files fetched per domain to count URLs and dates. 0 skips sitemaps.", "default": 25 }, "checkTerms": { "title": "Scan terms and legal pages for TDM clauses", "type": "boolean", "description": "Follows up to three terms/legal/policy links from the homepage and reports passages about text and data mining, AI training, automated access and reservations, with whether they prohibit.", "default": true }, "extraUserAgents": { "title": "Extra user-agent tokens", "maxItems": 50, "type": "array", "description": "Your own crawler tokens to evaluate against robots.txt, next to the built-in AI and search crawlers.", "items": { "type": "string", "minLength": 1, "maxLength": 100 }, "default": [] }, "includeRobotsText": { "title": "Include robots.txt text", "type": "boolean", "description": "Adds the robots.txt content (up to 20 kB) to each record.", "default": true }, "maxConcurrency": { "title": "Domains in parallel", "minimum": 1, "maximum": 20, "type": "integer", "description": "Each domain gets at most three requests per second regardless. Sitemap files over 4 MB (up to 32 MB each) are downloaded at most two at a time across all domains, and gzip sitemaps are unpacked one at a time: a higher value does not multiply sitemap memory, and does not speed up those large downloads. Smaller sitemap files, gzip ones included, are read in parallel like every other response and never wait for another domain's large download.", "default": 5 }, "webhookUrl": { "title": "Webhook URL", "type": "string", "description": "Receives a POST with a summary after each run that delivered records (in monitor mode: the changes). Must be a public http(s) address; redirects are not followed. Stored encrypted, because webhook URLs often carry a token." }, "stateKey": { "title": "State key", "pattern": "^[a-zA-Z0-9_-]{1,64}$", "type": "string", "description": "Separates independent monitors. Defaults to the task ID, so each saved task keeps its own baseline." }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Off by default on purpose: the audit measures how a site answers a declared bot from a data centre, which is how AI crawlers arrive. Switch on residential proxies only to audit a site that blocks Apify's addresses outright.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }