{ "openapi": "3.0.1", "info": { "title": "AI Crawler Content Visibility Auditor", "description": "Check whether AI crawler user agents can read public pages. Audit robots.txt, HTTP status, extracted text, required markers, and cloaking for each URL and bot.", "version": "0.5", "x-build-id": "zLJ4OybtbelmMeJe6" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/kingii98~ai-crawler-visibility-auditor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-kingii98-ai-crawler-visibility-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/kingii98~ai-crawler-visibility-auditor/runs": { "post": { "operationId": "runs-sync-kingii98-ai-crawler-visibility-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/kingii98~ai-crawler-visibility-auditor/run-sync": { "post": { "operationId": "run-sync-kingii98-ai-crawler-visibility-auditor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "urls" ], "properties": { "urls": { "title": "URLs to audit", "minItems": 1, "maxItems": 500, "type": "array", "description": "Public HTTP(S) page URLs to check. Private and reserved IP targets are rejected. Maximum 500 URLs.", "items": { "type": "string" }, "default": [ "https://example.com/" ] }, "botUserAgents": { "title": "Bot user agents", "minItems": 1, "maxItems": 20, "type": "array", "description": "User-Agent header values and robots.txt tokens for the AI crawlers to check. Optional; defaults to GPTBot, ClaudeBot, PerplexityBot, CCBot, Google-Extended, and Bingbot. Add or replace tokens as new AI bots appear. Maximum 20.", "items": { "type": "string" }, "default": [ "GPTBot", "ClaudeBot", "PerplexityBot", "CCBot", "Google-Extended", "Bingbot" ] }, "referenceUserAgent": { "title": "Reference user agent", "type": "string", "description": "User-Agent used for one baseline fetch per URL. Each bot fetch is compared against this baseline to detect cloaking.", "default": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" }, "requiredMarkers": { "title": "Required text markers", "maxItems": 20, "type": "array", "description": "Plain-text strings that must appear on the page, for example a price or a contact heading. Optional. Maximum 20.", "items": { "type": "string" }, "default": [] }, "maxBytesPerFetch": { "title": "Max bytes per fetch", "minimum": 1024, "maximum": 5000000, "type": "integer", "description": "Response body byte cap for each HTTP fetch. Bytes past the cap are dropped.", "default": 2000000 }, "concurrency": { "title": "Concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Maximum number of URLs audited at the same time.", "default": 10 }, "hostDelaySeconds": { "title": "Per-host delay (seconds)", "minimum": 0, "maximum": 5, "type": "number", "description": "Minimum delay between two requests sent to the same host.", "default": 0.2 }, "previousDatasetId": { "title": "Previous dataset ID", "type": "string", "description": "Optional. Dataset ID of an earlier run of this Actor. If set, the run reads that dataset and reports the URLs whose verdict changed since then. The dataset must be readable with the current Apify token." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }