{ "openapi": "3.0.1", "info": { "title": "Robots.txt and AI Crawler Access Checker API", "description": "Check which AI and search crawlers a website lets in: GPTBot, ClaudeBot, PerplexityBot, Google-Extended and more, read from robots.txt by the RFC 9309 rules. Also llms.txt, sitemaps and meta robots. Pay only per website checked.", "version": "1.0", "x-build-id": "XsZDsaGOv0PAq55aA" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rowcraft~robots-ai-crawler-checker/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rowcraft-robots-ai-crawler-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rowcraft~robots-ai-crawler-checker/runs": { "post": { "operationId": "runs-sync-rowcraft-robots-ai-crawler-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rowcraft~robots-ai-crawler-checker/run-sync": { "post": { "operationId": "run-sync-rowcraft-robots-ai-crawler-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "websites" ], "properties": { "websites": { "title": "Websites", "type": "array", "description": "Websites to check, one per line: a domain (example.com) or an address (https://www.example.com/shop). robots.txt is read from the top of each site; the path in an address is ignored (use \"Paths\" below). Duplicates and invalid entries are skipped and never charged.", "items": { "type": "string" } }, "paths": { "title": "Paths to check", "type": "array", "description": "The paths the access is calculated for, one per line, for example / or /blog/ or /products/shoes?page=2. A crawler counts as allowed only when every path is allowed. The default is the home page.", "default": [ "/" ], "items": { "type": "string" } }, "bots": { "title": "Crawlers to check", "type": "array", "description": "Leave empty to check all 27 listed crawlers (OpenAI, Anthropic, Perplexity, Google, Apple, Microsoft, Amazon, Meta, Mistral, DuckDuckGo, Common Crawl, Yandex, Baidu). Or list the names you want, for example GPTBot and ClaudeBot. Any other name is checked as your own user-agent token, and the word \"all\" adds the whole list to your own tokens.", "default": [], "items": { "type": "string" } }, "maxItems": { "title": "Maximum websites delivered", "minimum": 0, "type": "integer", "description": "Stop after this many rows are delivered. 0 = the whole list.", "default": 0 }, "onlyNew": { "title": "Only new rows since the last run", "type": "boolean", "description": "Remember the rows already delivered to you (in a key-value store in your Apify account) and deliver only rows that appeared since the last run with the same state key. The first run delivers everything that matches; later runs cost only the new rows. Ideal for schedules.", "default": false }, "stateName": { "title": "State name", "type": "string", "description": "Name of the memory used by \"Only new rows\". Use a different name for each schedule or search (for example `stockholm-python`) so they do not share one memory.", "default": "default" }, "notifyUrl": { "title": "Notify this URL when the run ends", "type": "string", "description": "A webhook address that receives one JSON POST with the number of new rows and links to the dataset: a Slack or Discord incoming webhook, a Make, n8n or Zapier catch hook, or your own endpoint. The message is also in the `text` (Slack) and `content` (Discord) fields." }, "notifyWhenEmpty": { "title": "Notify also when there are no new rows", "type": "boolean", "description": "By default the notification is sent only when at least one row was delivered.", "default": false }, "requestIntervalMs": { "title": "Delay between requests (ms)", "minimum": 250, "maximum": 10000, "type": "integer", "description": "Default 1000 ms (one request per second). Each website takes four requests: robots.txt, llms.txt, llms-full.txt and the home page (plus redirects), about five seconds at the default. Minimum 250.", "default": 1000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }