{ "openapi": "3.0.1", "info": { "title": "AI Crawler Checker - robots.txt Checker for GPTBot & AI Bots", "description": "Which AI bots can read a site? Bulk-check up to 10,000 domains for 26 AI crawlers (GPTBot, ClaudeBot, PerplexityBot...) in robots.txt: AI search score, fix snippet, Content Signals. $2/1k sites.", "version": "0.1", "x-build-id": "5ClgQJhNneyM82ezX" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/tidytools~ai-crawler-access-checker/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-tidytools-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/tidytools~ai-crawler-access-checker/runs": { "post": { "operationId": "runs-sync-tidytools-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/tidytools~ai-crawler-access-checker/run-sync": { "post": { "operationId": "run-sync-tidytools-ai-crawler-access-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "websites" ], "properties": { "websites": { "title": "Websites", "type": "array", "description": "Domains or URLs to check, one per line (lines with several domains separated by commas or spaces are split), e.g. example.com or https://www.example.com. robots.txt is read from the site root; a path in a URL (e.g. nytimes.com/section/world) is tested too. Lines that are not websites get an error row and are not charged. Up to 10,000 sites per run.", "items": { "type": "string" } }, "paths": { "title": "Paths to test", "type": "array", "description": "Optional paths to test for every bot, e.g. /blog/ or /products/. Default is the home page (/).", "items": { "type": "string" } }, "maxConcurrency": { "title": "Parallel sites", "minimum": 1, "maximum": 30, "type": "integer", "description": "How many websites are checked at the same time.", "default": 10 }, "includeFixSnippets": { "title": "Include robots.txt fix snippets", "type": "boolean", "description": "Add recommendedRobotsSnippet (ready-to-paste lines that re-allow blocked AI search and assistant crawlers) and suggestedPolicy (a full template that blocks training crawlers and allows AI search) to each result.", "default": true }, "blockedOnly": { "title": "Only output sites with a problem", "type": "boolean", "description": "Output only sites where an AI search or assistant crawler is blocked or partly blocked, or robots.txt could not be read. All sites are still checked and charged (except unreadable robots.txt).", "default": false }, "monitorName": { "title": "Monitor name (compare with the last run)", "type": "string", "description": "Optional. Runs with the same name compare each site with its previous result and add a \"comparison\" field (changed bots, policy, llms.txt). Unchanged sites are charged the re-check price ($0.50 / 1,000 instead of $2 / 1,000). Changing the paths to test does not count as a site change. Schedule the Actor weekly to track robots.txt changes." }, "outputOnlyChanges": { "title": "Only output changed sites", "type": "boolean", "description": "With a monitor name: skip sites whose AI crawler access did not change since the last run (new sites are still output). Skipped sites are still checked and charged the re-check price ($0.50 / 1,000).", "default": false }, "webhookUrl": { "title": "Webhook URL for changes", "type": "string", "description": "Optional. When at least one site changed, a summary is POSTed here. Slack and Discord webhook URLs get a formatted message; other URLs get JSON." }, "webhookSecret": { "title": "Webhook signing secret", "type": "string", "description": "Optional. Signs the webhook body with HMAC-SHA256 in the X-Signature-256 header (sha256=)." }, "httpVia": { "title": "Plain HTTP requests from", "enum": [ "auto", "backend", "direct", "vm" ], "type": "string", "description": "Some sites block requests from data centers. Auto retries from a second network. Used for the home page (noai check); robots.txt and llms.txt are always read from Apify's network. If both are refused, Auto also tries our second server (Oracle, different IP) before a real browser.", "default": "auto" }, "siteTimeoutSecs": { "title": "Time limit per website (seconds)", "minimum": 0, "maximum": 900, "type": "integer", "description": "A website whose robots.txt, llms.txt and home page take longer than this in total (all retries and fallbacks included) is reported with errorType \"timeout\" and not charged. 0 = no limit.", "default": 120 }, "proxyConfiguration": { "title": "Proxy (optional)", "type": "object", "description": "Only used for requests sent from Apify's network (robots.txt, llms.txt and the home page in direct mode). Apify proxy usage is billed to your Apify account." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }