{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor + Robots.txt & llms.txt Validator", "description": "Extract every URL from a site's sitemaps and check the sitemap files, robots.txt (rules for AI bots) and the structure of llms.txt. Streaming parser, flat memory; sites that cannot be read are free.", "version": "0.1", "x-build-id": "5twEO02YpX1EeZYvY" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/far003~sitemap-url-extractor-validator/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-far003-sitemap-url-extractor-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/far003~sitemap-url-extractor-validator/runs": { "post": { "operationId": "runs-sync-far003-sitemap-url-extractor-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/far003~sitemap-url-extractor-validator/run-sync": { "post": { "operationId": "run-sync-far003-sitemap-url-extractor-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "urls" ], "properties": { "urls": { "title": "Websites or sitemap URLs", "minItems": 1, "type": "array", "description": "Domains (example.com), site URLs (https://example.com/) or the URL of a sitemap file (https://example.com/sitemap.xml). One site report per entry. Entries that are not web addresses, or point to private/local addresses, are reported as \"invalid\" (free), not as a run error.", "items": { "type": "string" } }, "extractUrls": { "title": "Extract URLs", "type": "boolean", "description": "Deliver one record per URL found in the sitemaps. Turn off to get only the site reports (sitemap files, robots.txt, llms.txt, score): a sample of up to 50 sitemap files per site is read and validated (see Maximum sitemap files per site); the URLs are not delivered or charged.", "default": true }, "maxUrlsPerSite": { "title": "Maximum URLs per site (0 = no limit)", "minimum": 0, "type": "integer", "description": "Stop reading a site after this many URL records were delivered. The site report then says that the count of URLs found is a lower bound.", "default": 0 }, "maxSitemapsPerSite": { "title": "Maximum sitemap files per site (0 = automatic)", "minimum": 0, "maximum": 5000, "type": "integer", "description": "How many sitemap files of one site are read. 0 = automatic: up to 5,000 when the URLs are extracted, a sample of 50 files when \"Extract URLs\" is off (the cheap site-report mode). When the limit is reached the report says \"N of M sitemap files checked\", is marked incomplete, and loses no score points.", "default": 0 }, "includePatterns": { "title": "Include patterns", "type": "array", "description": "Only deliver URLs matching one of these patterns. `*` matches within one path segment, `**` matches anything. A pattern starting with `/` is compared with the path and query (`/blog/**`), any other with the whole URL (`https://example.com/blog/*`). Empty = all URLs.", "default": [], "items": { "type": "string" } }, "excludePatterns": { "title": "Exclude patterns", "type": "array", "description": "Do not deliver URLs matching one of these patterns (same syntax as the include patterns). Exclusion wins over inclusion. Filtered URLs are still counted and checked in the site report.", "default": [], "items": { "type": "string" } }, "modifiedAfter": { "title": "Only URLs modified after", "type": "string", "description": "A date (2026-01-31) or a number of days (\"30 days\"). URLs with a lastmod before it, and URLs without a valid lastmod, are not delivered." }, "onlyNew": { "title": "Only new or changed URLs", "type": "boolean", "description": "Deliver only URLs that are new, or whose lastmod changed, since the previous run with this option. The first run delivers everything as new. Only URLs that were actually delivered are remembered: filters and limits never make the Actor forget or invent anything.", "default": false }, "checkRobotsTxt": { "title": "Check robots.txt", "type": "boolean", "description": "Read and validate robots.txt (RFC 9309), build the table for AI bots, and mark every URL with robotsAllowed and aiBotsBlocked.", "default": true }, "checkLlmsTxt": { "title": "Check llms.txt", "type": "boolean", "description": "Read /llms.txt (and look for /llms-full.txt) and validate the structure against the llmstxt.org proposal.", "default": true }, "extraUserAgents": { "title": "Extra bots for the robots.txt table", "type": "array", "description": "Bot names (user-agent tokens, e.g. MyCompanyBot) to add to the table of AI bots, in the site report and in aiBotsBlocked.", "default": [], "items": { "type": "string" } }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Do not fetch a sitemap or llms.txt that robots.txt forbids to this Actor's user-agent (SitemapRobotsLlmsBot). If robots.txt answers with a server error, RFC 9309 says to assume everything is disallowed.", "default": true }, "requestsPerSecond": { "title": "Requests per second, per host", "minimum": 1, "maximum": 5, "type": "integer", "description": "Between 1 and 5. A Crawl-delay in robots.txt that is slower than this wins (capped at 10 seconds).", "default": 2 }, "maxConcurrency": { "title": "Sites in parallel", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many sites are read at the same time.", "default": 5 }, "timeoutSecs": { "title": "Timeout per request (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Maximum wait for the response headers, and maximum silence while a file is downloading.", "default": 30 }, "maxSitemapMegabytes": { "title": "Maximum size of a sitemap (MB, uncompressed)", "minimum": 1, "maximum": 500, "type": "integer", "description": "A sitemap larger than this is read up to the limit and reported as truncated_at_limit. The sitemap standard allows 50 MB.", "default": 50 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }