{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor", "description": "List every URL a website publishes in its XML sitemaps, found automatically through robots.txt. Filter by text or lastmod date, or get only the URLs added since your last run.", "version": "0.1", "x-build-id": "QJ2M0LRmnQPn6Rhg1" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/sampon~sitemap-url-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-sampon-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/sampon~sitemap-url-extractor/runs": { "post": { "operationId": "runs-sync-sampon-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/sampon~sitemap-url-extractor/run-sync": { "post": { "operationId": "run-sync-sampon-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemap URLs", "type": "array", "description": "A website home page (for example https://crawlee.dev/) or the address of a sitemap or sitemap index. Several can be listed in one run.", "default": [ { "url": "https://crawlee.dev/" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxItems": { "title": "Maximum URLs", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Stop after saving this many URLs in total. Raise it to get the whole sitemap.", "default": 50 }, "includePatterns": { "title": "Keep URLs containing", "type": "array", "description": "Keep only URLs that contain at least one of these texts, such as /blog/ or /products/. Case does not matter. Leave empty to keep everything.", "items": { "type": "string" } }, "excludePatterns": { "title": "Drop URLs containing", "type": "array", "description": "Drop URLs that contain any of these texts, such as ?page= or /tag/.", "items": { "type": "string" } }, "patternsAreRegex": { "title": "Treat the texts above as regular expressions", "type": "boolean", "description": "Turn on to write the two lists above as regular expressions (Python syntax, case-insensitive), for example /p/\\d+$.", "default": false }, "lastmodSince": { "title": "Changed on or after", "type": "string", "description": "Keep only URLs whose lastmod date is on or after this point. Write a date such as 2026-01-31, or a period such as 7 days, 2 weeks or 3 months. Leave empty to ignore dates." }, "keepUrlsWithoutLastmod": { "title": "Keep URLs without a lastmod date", "type": "boolean", "description": "When a date filter is set, URLs that have no lastmod in the sitemap are dropped. Turn this on to keep them.", "default": false }, "onlyNewUrls": { "title": "Only URLs added since the last run", "type": "boolean", "description": "Remembers the URLs seen for each website and returns only URLs that were not there last time. The first run returns everything and sets the baseline. Every sitemap is read in this mode, even after the maximum is reached, so the memory stays complete.", "default": false }, "rememberUrls": { "title": "Remember URLs and mark new ones", "type": "boolean", "description": "Keeps all rows but fills the isNew column (true or false) by comparing with earlier runs. Not needed when the option above is on.", "default": false }, "stateStoreName": { "title": "Memory name", "type": "string", "description": "Name of the key-value store in your account that holds the remembered URLs. Use a different name to keep separate histories, for example one per project.", "default": "sitemap-url-extractor-state" }, "maxSitemaps": { "title": "Maximum sitemap files", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Safety limit on how many sitemap files are read per run, including index files.", "default": 500 }, "maxConcurrency": { "title": "Parallel requests", "minimum": 1, "maximum": 4, "type": "integer", "description": "How many sitemap files to download at the same time. Capped at 4 to stay gentle on the website.", "default": 2 }, "requestDelayMs": { "title": "Gap between requests (ms)", "minimum": 250, "maximum": 10000, "type": "integer", "description": "Minimum time between two requests to the same website. A longer crawl delay in the site's robots.txt always wins.", "default": 500 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }