{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor — All Page URLs from a Website", "description": "Input: domains or sitemap URLs. Output: one row per page URL with its lastmod, changefreq and priority, plus a report row per sitemap file read. Finds sitemaps via robots.txt, follows nested indexes, unpacks gzip. Filter by URL pattern or by date. Hard cap of 25,000 URLs per run.", "version": "1.0", "x-build-id": "zAer87t6q0V2Aq34E" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/power_on~sitemap-url-list/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-power_on-sitemap-url-list", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/power_on~sitemap-url-list/runs": { "post": { "operationId": "runs-sync-power_on-sitemap-url-list", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/power_on~sitemap-url-list/run-sync": { "post": { "operationId": "run-sync-power_on-sitemap-url-list", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemap URLs", "minItems": 1, "type": "array", "description": "One per line. Give a bare domain (example.com), a homepage, or a direct link to a sitemap.xml / sitemap_index.xml / sitemap.xml.gz. For a domain, the Actor reads robots.txt and tries the common sitemap paths. One set of results is produced per entry.", "default": [ "https://apify.com" ], "items": { "type": "string" } }, "maxUrls": { "title": "Max URLs per website", "minimum": 0, "maximum": 25000, "type": "integer", "description": "Stop after this many URLs for a single website. Set it to 0 to go up to the run limit. Every run returns at most 25,000 URLs in total; when it stops there, a report row says so and lists the sitemap files still to read, so you can run those directly. The default keeps a first run small and predictable — raise it once you know the site.", "default": 1000 }, "includeRegex": { "title": "Include only URLs matching", "type": "string", "description": "A JavaScript regular expression. When set, only URLs that match are kept, e.g. ^https://example\\.com/blog/ for one section." }, "excludeRegex": { "title": "Exclude URLs matching", "type": "string", "description": "A JavaScript regular expression. URLs that match are dropped, e.g. \\.(pdf|jpg|png)$ or /tag/ ." }, "lastmodAfter": { "title": "Only URLs changed on or after", "type": "string", "description": "A date (YYYY-MM-DD). Keeps only URLs whose is on or after it. URLs without a are dropped when this is set." }, "flatten": { "title": "One row per URL", "type": "boolean", "description": "On (default): the dataset has one row per URL, ready to pipe into a crawler or the Website Screenshot Actor. Off: one row per website, with the URLs as an array and a per-sitemap breakdown.", "default": true }, "includeSitemapReport": { "title": "Include the sitemap report", "type": "boolean", "description": "Add a final row per website listing every sitemap found, where it was discovered, its status and how many URLs it contributed. This is what lets you see why a count is lower than expected.", "default": true }, "reportOnly": { "title": "Report only - count the pages, do not return them", "type": "boolean", "description": "Return only the sitemap report: one row per sitemap file, plus a row with the total number of pages the site lists. No URLs are returned and nothing is charged. Use it to audit a site's sitemaps or to count its pages before deciding how many URLs to extract. With this on, \"Max URLs per website\" is ignored, so the report covers every sitemap file instead of stopping at the first few.", "default": false }, "maxSitemaps": { "title": "Max sitemap files per website", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Safety limit on how many sitemap files (including those linked from a sitemap index) to fetch for one website.", "default": 200 }, "timeoutSecs": { "title": "Timeout per request (s)", "minimum": 5, "maximum": 120, "type": "integer", "description": "How long to wait for a single sitemap or robots.txt file before giving up on it and recording the reason.", "default": 30 }, "extraHeaders": { "title": "Extra HTTP headers", "type": "object", "description": "Extra HTTP headers sent with every request, as a JSON object. Use it for an Authorization header or a custom User-Agent." }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional proxy, for sites that block datacenter traffic or serve a region-specific sitemap. Not needed for most sites.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }