{ "openapi": "3.0.1", "info": { "title": "AI Crawler Access Checker: Is GPTBot or ClaudeBot Blocked?", "description": "Shows if GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot and Meta-ExternalAgent are allowed or blocked on each site in a dataset, CSV or Sheet: robots.txt, a real fetch per bot, llms.txt, plus change monitoring. Inputs: datasetId, fileUrl, urls. $0.005 per URL. Agent-ready: x402, MCP.", "version": "0.1", "x-build-id": "WXWN11lIJbxQF0woq" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/nerolabs~ai-crawler-access-checker/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-nerolabs-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/nerolabs~ai-crawler-access-checker/runs": { "post": { "operationId": "runs-sync-nerolabs-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/nerolabs~ai-crawler-access-checker/run-sync": { "post": { "operationId": "run-sync-nerolabs-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetId": { "title": "Dataset", "type": "string", "description": "An Apify dataset with one row per website, for example a Google Maps or lead scraper's output. Every original column is kept and the AI crawler verdicts are added alongside. Use the picker rather than typing an ID." }, "fileUrl": { "title": "File or Google Sheet URL", "type": "string", "description": "A public link to a CSV, TSV, Excel, JSON or JSON Lines file with one row per website. A normal Google Sheets link works: share it as 'Anyone with the link can view'. Used when no dataset is given." }, "fileFormat": { "title": "File format", "enum": [ "auto", "csv", "tsv", "json", "jsonl", "xlsx" ], "type": "string", "description": "Leave on 'Detect automatically' unless the link has no file extension and the server reports the wrong content type.", "default": "auto" }, "sheetName": { "title": "Excel sheet name", "type": "string", "description": "Which sheet to read from an Excel workbook. Defaults to the first sheet." }, "urls": { "title": "Website URLs", "type": "array", "description": "A plain list of websites for a quick run with no spreadsheet (bare domains like example.com are fine). The URL you give is the page tested, so use the homepage unless you care about a specific path. Use the dataset, file or Google Sheet inputs above to keep your own columns next to the results.", "items": { "type": "string" } }, "data": { "title": "Rows", "type": "array", "description": "Rows as inline JSON, an alternative to a dataset or file. Each object needs a field holding the website." }, "urlField": { "title": "Website field", "type": "string", "description": "The column holding the website to check. Left empty it is detected automatically, preferring columns named website, domain, homepage or url." }, "bots": { "title": "AI crawlers to check", "type": "array", "description": "Which crawlers to test. Default: GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot and Meta-ExternalAgent. Google-Extended and Applebot-Extended are robots.txt control tokens with no crawler of their own, so only their robots.txt rule is checked.", "items": { "type": "string", "enum": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Meta-ExternalAgent", "Google-Extended", "Applebot-Extended", "CCBot" ], "enumTitles": [ "GPTBot (OpenAI training)", "OAI-SearchBot (ChatGPT search)", "ChatGPT-User (ChatGPT opening a page for a user)", "ClaudeBot (Anthropic training)", "PerplexityBot (Perplexity search)", "Meta-ExternalAgent (Meta AI training)", "Google-Extended (Gemini, robots.txt only)", "Applebot-Extended (Apple AI, robots.txt only)", "CCBot (Common Crawl)" ] }, "default": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Meta-ExternalAgent" ] }, "testRealFetch": { "title": "Test a real fetch per crawler", "type": "boolean", "description": "Besides reading robots.txt, request the URL once per crawler with its published user agent and once as a normal browser, to catch firewalls and CDNs that refuse AI crawlers even though robots.txt allows them. About six requests per site. Turn off for a robots.txt-only check.", "default": true }, "checkLlmsTxt": { "title": "Check for llms.txt", "type": "boolean", "description": "Also look for /llms.txt and /llms-full.txt on each site. Free, two extra requests per site.", "default": true }, "monitor": { "title": "Monitor mode", "type": "boolean", "description": "Remember each site's result and report what changed since the last run (a crawler newly blocked or unblocked, llms.txt added or removed). Run it on a schedule. The first run of a site just records its starting point. Off by default.", "default": false }, "monitorName": { "title": "Monitor name", "type": "string", "description": "Names the saved memory for monitor mode, so different watchlists do not mix. Use the same name on every scheduled run of one watchlist. Default: 'default'." }, "keep": { "title": "Which rows to keep", "enum": [ "all", "blocked", "changed" ], "type": "string", "description": "Filtering happens after each site is checked. 'Only changed sites' is the monitor-mode alert list.", "default": "all" }, "keepOriginalFields": { "title": "Keep my original columns", "type": "boolean", "description": "Keep every column from the input row next to the results, so they line up with your own data.", "default": true }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Off by default and usually not needed. Requests then come from an Apify data centre. Turning a proxy on changes which address the sites see, which can change firewall results.", "default": { "useApifyProxy": false } }, "concurrency": { "title": "Sites at a time", "minimum": 1, "maximum": 30, "type": "integer", "description": "How many sites to check in parallel. Never more than two requests at a time go to any one site.", "default": 10 }, "requestTimeoutSecs": { "title": "Request timeout", "minimum": 5, "maximum": 60, "type": "integer", "description": "How long to wait for one request before giving up. A site that never answers is never charged.", "default": 15 }, "maxItems": { "title": "Maximum rows", "minimum": 1, "maximum": 50000, "type": "integer", "description": "A hard ceiling on how many rows are read from the input, as a safety net on a large dataset." }, "exportFormats": { "title": "Export files", "type": "array", "description": "Also write the results as a downloadable CSV or Excel file, linked from the run's output. $0.01 per file, charged only when the run checked at least one URL.", "items": { "type": "string", "enum": [ "csv", "xlsx" ], "enumTitles": [ "CSV", "Excel (xlsx)" ] }, "default": [] }, "outputDatasetName": { "title": "Append to named dataset", "type": "string", "description": "Also append every kept row to a named dataset that persists across runs, for example a history of checks. Not charged again." }, "webhookUrl": { "title": "Webhook URL", "type": "string", "description": "POST the run summary to this URL when the run finishes, for Slack, Zapier, Make, n8n or your own API. In monitor mode it is sent only when something changed. Charged $0.01 only on a confirmed 2xx response." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }