{
  "openapi": "3.0.1",
  "info": {
    "title": "AI crawler access checker — GPTBot ClaudeBot robots",
    "description": "Check whether major AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, OAI-SearchBot, …) are allowed by robots.txt — and flag likely WAF/CDN blocks on sample URLs.",
    "version": "0.1",
    "x-build-id": "iucsBQRt4AynYjlws"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/kondasviktor~ai-crawler-access-checker/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-kondasviktor-ai-crawler-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/kondasviktor~ai-crawler-access-checker/runs": {
      "post": {
        "operationId": "runs-sync-kondasviktor-ai-crawler-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/kondasviktor~ai-crawler-access-checker/run-sync": {
      "post": {
        "operationId": "run-sync-kondasviktor-ai-crawler-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "sites"
        ],
        "properties": {
          "sites": {
            "title": "Websites",
            "type": "array",
            "description": "Domains or URLs, one per line: <code>example.com</code> or <code>https://example.com/blog/</code>. One result per site (scheme + host: <code>example.com</code> and <code>www.example.com</code> are different sites). A path in a URL is checked as an extra sample path for that site. Public sites only.",
            "items": {
              "type": "string"
            }
          },
          "bots": {
            "title": "AI crawlers to check",
            "type": "array",
            "description": "Which robots.txt user-agent tokens to evaluate. Leave empty for the default set (the major AI training, search, user-fetch and ads crawlers). <b>Google-Extended</b> and <b>Applebot-Extended</b> are control tokens: they never crawl, robots.txt is the only place they are read. Optional entries have not been verified against the operator’s documentation.",
            "items": {
              "type": "string",
              "enum": [
                "GPTBot",
                "OAI-SearchBot",
                "ChatGPT-User",
                "OAI-AdsBot",
                "ClaudeBot",
                "Claude-SearchBot",
                "Claude-User",
                "PerplexityBot",
                "Perplexity-User",
                "Google-Extended",
                "Applebot-Extended",
                "CCBot",
                "meta-externalagent",
                "meta-externalfetcher",
                "Applebot",
                "Amazonbot",
                "Bytespider",
                "DuckAssistBot",
                "MistralAI-User"
              ],
              "enumTitles": [
                "GPTBot — OpenAI (training)",
                "OAI-SearchBot — OpenAI (search)",
                "ChatGPT-User — OpenAI (user-fetch)",
                "OAI-AdsBot — OpenAI (ads)",
                "ClaudeBot — Anthropic (training)",
                "Claude-SearchBot — Anthropic (search)",
                "Claude-User — Anthropic (user-fetch)",
                "PerplexityBot — Perplexity (search)",
                "Perplexity-User — Perplexity (user-fetch)",
                "Google-Extended — Google (training, control token)",
                "Applebot-Extended — Apple (training, control token)",
                "CCBot — Common Crawl (training)",
                "meta-externalagent — Meta (training)",
                "meta-externalfetcher — Meta (user-fetch)",
                "Applebot — Apple (search, optional)",
                "Amazonbot — Amazon (other, optional)",
                "Bytespider — ByteDance (training, optional)",
                "DuckAssistBot — DuckDuckGo (search, optional)",
                "MistralAI-User — Mistral (user-fetch, optional)"
              ]
            },
            "default": [
              "GPTBot",
              "OAI-SearchBot",
              "ChatGPT-User",
              "OAI-AdsBot",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "PerplexityBot",
              "Perplexity-User",
              "Google-Extended",
              "Applebot-Extended",
              "CCBot",
              "meta-externalagent",
              "meta-externalfetcher"
            ]
          },
          "customBots": {
            "title": "Extra bot tokens",
            "type": "array",
            "description": "Any other robots.txt user-agent token, e.g. a crawler announced after this Actor was published. Letters, digits, <code>.</code> <code>_</code> <code>-</code> only.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "samplePaths": {
            "title": "Sample paths",
            "type": "array",
            "description": "Paths to evaluate for every bot, e.g. <code>/</code>, <code>/blog/</code>, <code>/products/</code>. A bot that may crawl some of them but not all gets the verdict <code>partial</code>. Maximum 10 per site, including paths typed inside the site URLs.",
            "default": [
              "/"
            ],
            "items": {
              "type": "string"
            }
          },
          "checkLlmsTxt": {
            "title": "Check /llms.txt",
            "type": "boolean",
            "description": "One extra request per site: is there an <code>/llms.txt</code>, and does it look like Markdown with a title and links? (Presence and shape only — not a full validation.)",
            "default": true
          },
          "maxSites": {
            "title": "Max sites",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "Hard cap on paid results — sites whose robots.txt was read. Free rows (unreachable, blocked, invalid) do not count. This is also your cost cap: 100 sites = $0.50.",
            "default": 100
          },
          "wafProbe": {
            "title": "WAF / CDN probe (advanced, opt-in)",
            "type": "boolean",
            "description": "<b>Only for sites you own or are allowed to test.</b> Requests up to 3 sample URLs once with a neutral user-agent and once per probe bot with that bot’s User-Agent token, and compares status codes and challenge-page markers. It shows whether a <i>user-agent rule</i> or a challenge page treats the bot differently. It runs from an Apify datacenter IP, so it <b>cannot prove</b> what the real crawler (which CDNs may verify by IP range) experiences. Adds up to ~30 requests per site; no extra charge.",
            "default": false
          },
          "probeBots": {
            "title": "Probe bots",
            "type": "array",
            "description": "Crawlers to probe when the WAF probe is on (maximum 8; control tokens cannot be probed). The Actor sends an approximate User-Agent string containing the bot’s token — not the vendor’s exact string.",
            "items": {
              "type": "string",
              "enum": [
                "GPTBot",
                "OAI-SearchBot",
                "ChatGPT-User",
                "OAI-AdsBot",
                "ClaudeBot",
                "Claude-SearchBot",
                "Claude-User",
                "PerplexityBot",
                "Perplexity-User",
                "CCBot",
                "meta-externalagent",
                "meta-externalfetcher",
                "Applebot",
                "Amazonbot",
                "Bytespider",
                "DuckAssistBot",
                "MistralAI-User"
              ],
              "enumTitles": [
                "GPTBot — OpenAI",
                "OAI-SearchBot — OpenAI",
                "ChatGPT-User — OpenAI",
                "OAI-AdsBot — OpenAI",
                "ClaudeBot — Anthropic",
                "Claude-SearchBot — Anthropic",
                "Claude-User — Anthropic",
                "PerplexityBot — Perplexity",
                "Perplexity-User — Perplexity",
                "CCBot — Common Crawl",
                "meta-externalagent — Meta",
                "meta-externalfetcher — Meta",
                "Applebot — Apple",
                "Amazonbot — Amazon",
                "Bytespider — ByteDance",
                "DuckAssistBot — DuckDuckGo",
                "MistralAI-User — Mistral"
              ]
            },
            "default": [
              "GPTBot",
              "OAI-SearchBot",
              "ChatGPT-User",
              "ClaudeBot",
              "PerplexityBot"
            ]
          },
          "respectRobots": {
            "title": "Respect robots.txt for this Actor’s own requests",
            "type": "boolean",
            "description": "On (recommended): <code>/llms.txt</code> and probe URLs are skipped when the site’s robots.txt disallows this Actor’s own user-agent (<code>AICrawlerAccessChecker</code>, falling back to <code>*</code>). Turn off only for sites you own or may test. Reading <code>/robots.txt</code> itself is always done.",
            "default": true
          },
          "maxConcurrency": {
            "title": "Sites in parallel",
            "minimum": 1,
            "maximum": 5,
            "type": "integer",
            "description": "How many different sites are checked at the same time (1–5). Requests to one site are always sequential with a pause of at least 0.7 s.",
            "default": 3
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (seconds)",
            "minimum": 5,
            "maximum": 60,
            "type": "integer",
            "description": "Per request. A site that does not answer in time is reported as unreachable (free).",
            "default": 15
          },
          "trackHistory": {
            "title": "Track changes between runs",
            "type": "boolean",
            "description": "Remember each site’s verdicts in a named key-value store in <b>your</b> account and add <code>change</code> / <code>changeDetail</code> to every result on the next run. Schedule the Actor weekly to get policy-change alerts.",
            "default": true
          },
          "historyStoreName": {
            "title": "History store name",
            "type": "string",
            "description": "Named key-value store that holds the change-tracking state. Use different names to keep separate watchlists apart.",
            "default": "ai-crawler-access-history"
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}