{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & Scraper - All Page URLs of a Site", "description": "Sitemap scraper and URL extractor: give it a website or sitemap.xml and get every page URL with lastmod, changefreq and priority. Finds sitemaps via robots.txt, follows sitemap indexes, reads .xml.gz and text sitemaps, survives broken XML. Plain HTTP, no browser. $0.20 per 1,000 URLs.", "version": "0.1", "x-build-id": "m7Occ8r2diwfeATbO" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/datagleaner~sitemap-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-datagleaner-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/datagleaner~sitemap-extractor/runs": { "post": { "operationId": "runs-sync-datagleaner-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/datagleaner~sitemap-extractor/run-sync": { "post": { "operationId": "run-sync-datagleaner-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "websites": { "title": "Websites", "type": "array", "description": "Website URLs or bare domains, for example https://aioseo.com or stripe.com. Only the site root is used; any path is ignored. A direct sitemap URL (ending in .xml or .xml.gz) is also accepted.", "items": { "type": "string" } }, "maxUrlsPerSite": { "title": "Max URLs per site", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Upper bound on page URLs returned for each website. Each returned URL is billed, so this is also your cost cap per site. The status message names every site that had more.", "default": 5000 }, "checkStatus": { "title": "Check HTTP status of each page", "type": "boolean", "description": "Sends one HEAD request per URL (GET if HEAD is refused) and adds status, finalUrl (after redirects) and checkError, so you can find 404s, 5xx errors and redirects listed in the sitemap. When the sitemap has no lastmod, it is filled from the page's Last-Modified header (lastmodSource: http-header). Slower: about 5 pages a second.", "default": false }, "includeUrlPatterns": { "title": "Only include URLs matching", "type": "array", "description": "Optional regular expressions (Python syntax). A URL is kept only if it matches at least one. Example: /blog/", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URLs matching", "type": "array", "description": "Optional regular expressions. A URL matching any of them is dropped. Example: \\.pdf$", "items": { "type": "string" } }, "requestDelaySeconds": { "title": "Delay between requests (seconds)", "minimum": 0, "maximum": 30, "type": "number", "description": "Base pause between requests to the same site; the actual pause is randomised up to 1.5x. Raise it if a site throttles you.", "default": 0.3 }, "maxCrawlingDepth": { "title": "Max sitemap index depth", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many levels of sitemap indexes to follow. 1 = follow only the top-level index (its child sitemaps are read, but indexes nested inside it are not).", "default": 6 }, "maxRequestRetries": { "title": "Max request retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Retries per sitemap file on timeouts, 429 and 5xx errors (exponential backoff).", "default": 3 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Most sites work without a proxy; enable Apify Proxy if a site blocks datacenter IPs." }, "startUrls": { "title": "Start URLs (alternative to Websites)", "type": "array", "description": "Same as Websites, in the request-list format other sitemap Actors use, so their input works here unchanged. Used only when Websites is empty.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }