{
"openapi": "3.0.1",
"info": {
"title": "Sitemap Extractor + robots.txt & llms.txt Checker for AI",
"description": "Extract every URL from a website's sitemaps (indexes, gzip, nested) and check robots.txt and llms.txt: which AI crawlers (GPTBot, ClaudeBot, Google-Extended) may read the site, with the deciding rule. For SEO audits, crawler seeding and AI agents. $0.002/site.",
"version": "1.0",
"x-build-id": "aaQjYG3wP2BGqIsNg"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/tinplatelabs~site-discovery/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-tinplatelabs-site-discovery",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/tinplatelabs~site-discovery/runs": {
"post": {
"operationId": "runs-sync-tinplatelabs-site-discovery",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/tinplatelabs~site-discovery/run-sync": {
"post": {
"operationId": "run-sync-tinplatelabs-site-discovery",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"properties": {
"sites": {
"title": "Sites",
"type": "array",
"description": "Sites to analyze, one per line (every sitemap URL, robots.txt access for AI crawlers, llms.txt): a domain (example.com) or any URL on the site (https://www.example.com/blog/). Without a scheme, HTTPS is tried first, then HTTP. Duplicates are analyzed once. Max 1,000 per run. Used when mode is discover.",
"items": {
"type": "string"
}
},
"mode": {
"title": "Mode",
"enum": [
"discover",
"check"
],
"type": "string",
"description": "discover: one full record per site (robots.txt, AI-crawler access, sitemaps and URLs, llms.txt). check: one record per URL in URLs to check saying whether each user agent may fetch it according to robots.txt, and which rule decided.",
"default": "discover"
},
"urlsToCheck": {
"title": "URLs to check (check mode)",
"type": "array",
"description": "Full URLs to test against their site's robots.txt, e.g. https://example.com/blog/post-1. Each site's robots.txt is fetched once. Max 10,000 per run. Used when mode is check.",
"items": {
"type": "string"
}
},
"userAgents": {
"title": "User agents to evaluate",
"type": "array",
"description": "Robots.txt product tokens to evaluate (case-insensitive), e.g. GPTBot, ClaudeBot, Google-Extended, *. Leave empty for the default list of AI and search crawlers: GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, anthropic-ai, Google-Extended, Googlebot, Bingbot, PerplexityBot, Perplexity-User, CCBot, Applebot-Extended, Bytespider, meta-externalagent, Amazonbot, cohere-ai and *.",
"items": {
"type": "string"
}
},
"includeUrls": {
"title": "Return the URL list",
"type": "boolean",
"description": "Return the URLs found in the sitemaps (with lastmod, changefreq, priority). Turn off to get counts only (up to 1,000,000 URLs per site), which costs no URL fees.",
"default": true
},
"maxUrls": {
"title": "Max URLs per site",
"minimum": 1,
"maximum": 200000,
"type": "integer",
"description": "Stop after this many matching URLs per site; the record is then marked incomplete. URLs are billed per 1,000 returned, so this also caps the cost per site.",
"default": 10000
},
"urlPrefixes": {
"title": "Only URLs starting with",
"type": "array",
"description": "Keep only URLs that start with one of these. A value starting with / is matched against the URL path (/blog/); a full URL is matched against the whole URL (https://shop.example.com/).",
"items": {
"type": "string"
}
},
"urlRegex": {
"title": "Only URLs matching regex",
"type": "string",
"description": "Keep only URLs where this Python regular expression matches anywhere, e.g. /(docs|guides)/."
},
"excludeUrlRegex": {
"title": "Exclude URLs matching regex",
"type": "string",
"description": "Drop URLs where this Python regular expression matches anywhere, e.g. \\.(pdf|jpg)$|/tag/."
},
"maxSitemaps": {
"title": "Max sitemaps per site",
"minimum": 1,
"maximum": 2000,
"type": "integer",
"description": "Read at most this many sitemap files per site (indexes count too). Very large sites can have thousands; the record says how many were left unread.",
"default": 200
},
"maxConcurrency": {
"title": "Sites in parallel",
"minimum": 1,
"maximum": 20,
"type": "integer",
"description": "How many sites are analyzed at the same time. Requests to any one host are always paced (Crawl-delay honoured up to 3 s).",
"default": 5
},
"siteTimeoutSecs": {
"title": "Time limit per site (seconds)",
"minimum": 30,
"maximum": 1800,
"type": "integer",
"description": "When reading sitemaps takes longer than this, the URLs found so far are returned and the record is marked incomplete (still a success).",
"default": 300
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}