{ "openapi": "3.0.1", "info": { "title": "Sitemap Extractor – All URLs, lastmod, Nested & .gz", "description": "Extract every page URL from a website's XML sitemaps with lastmod, changefreq, priority and image counts. Finds sitemaps through robots.txt automatically and handles nested indexes and .xml.gz. Optional mode returns only URLs new since the last run.", "version": "1.0", "x-build-id": "cngxH7BGqWuezT1Mi" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/appetizing_horsefly~sitemap-url-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-appetizing_horsefly-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/appetizing_horsefly~sitemap-url-extractor/runs": { "post": { "operationId": "runs-sync-appetizing_horsefly-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/appetizing_horsefly~sitemap-url-extractor/run-sync": { "post": { "operationId": "run-sync-appetizing_horsefly-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sites" ], "properties": { "sites": { "title": "Websites or sitemap URLs", "type": "array", "description": "Domains (sitemaps found automatically via robots.txt, /sitemap.xml, /sitemap_index.xml) or direct sitemap URLs, including .xml.gz and plain-text sitemaps.", "default": [ "https://crawlee.dev", "https://www.allbirds.com/sitemap.xml" ], "items": { "type": "string" } }, "includePatterns": { "title": "Include URL regexes", "type": "array", "description": "Keep only URLs matching ANY of these regexes, e.g. /blog/ or /products/. Empty = all.", "items": { "type": "string" } }, "excludePatterns": { "title": "Exclude URL regexes", "type": "array", "description": "Drop URLs matching any of these regexes.", "items": { "type": "string" } }, "lastmodAfter": { "title": "Changed after (date)", "type": "string", "description": "Keep only URLs whose lastmod is on or after this date (YYYY-MM-DD). URLs without lastmod are kept unless 'Require lastmod' is on." }, "requireLastmod": { "title": "Require lastmod", "type": "boolean", "description": "Drop URLs that have no lastmod when a date filter is set.", "default": false }, "onlyNewSinceLastRun": { "title": "Only new URLs since last run", "type": "boolean", "description": "Remember URLs from previous runs (named key-value store) and output only new ones - a new-page / new-product monitor.", "default": false }, "stateStoreName": { "title": "State store name", "type": "string", "description": "Named key-value store for 'only new' memory.", "default": "sitemap-urls-state" }, "maxUrls": { "title": "Max URLs (hard cap)", "minimum": 0, "type": "integer", "description": "Stop after this many output URLs. You are never billed for more. 0 = no cap.", "default": 200 }, "maxSitemaps": { "title": "Max sitemap files per site", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Safety cap on nested sitemap files fetched per site.", "default": 500 }, "requestTimeoutSecs": { "title": "Request timeout (s)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Per-request timeout; retried 3 times.", "default": 30 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }