{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & Change Monitor", "description": "Sitemap URL extractor that reads robots.txt, sitemap indexes, .xml.gz and plain-text sitemaps and returns one row per URL with lastmod, changefreq, priority — plus a new/removed diff between runs.", "version": "0.1", "x-build-id": "X8ShqB6CVuf5xMBBV" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/webdatatools~sitemap-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-webdatatools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/webdatatools~sitemap-extractor/runs": { "post": { "operationId": "runs-sync-webdatatools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/webdatatools~sitemap-extractor/run-sync": { "post": { "operationId": "run-sync-webdatatools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sources" ], "properties": { "sources": { "title": "Sitemaps or domains", "type": "array", "description": "Enter the sitemaps to read, or just the domains, e.g. https://apify.com or https://www.allbirds.com/sitemap.xml. For a bare domain the Actor reads the Sitemap: lines of /robots.txt and falls back to /sitemap.xml, /sitemap_index.xml, /sitemap-index.xml, /sitemap.xml.gz and /sitemap.txt. Sitemap indexes are followed into their child sitemaps automatically.", "default": [ "https://apify.com" ], "items": { "type": "string" } }, "maxUrlsPerSource": { "title": "Max URLs per source", "minimum": 1, "maximum": 200000, "type": "integer", "description": "Enter how many URLs to keep per source, e.g. 5000. Counted across all child sitemaps of an index, so a 200,000-URL e-commerce site stops as soon as the cap is reached. Each URL is one billed dataset row.", "default": 1000 }, "urlFilter": { "title": "URL filter (regex)", "type": "string", "description": "Optional JavaScript regular expression; only URLs matching it are kept, e.g. /products/ for a Shopify catalogue or \\.pdf$ for documents. Leave empty to keep every URL. The pattern is matched against the full URL and is case-sensitive, so write [Pp]roducts when you need both cases.", "default": "" }, "mode": { "title": "Mode", "enum": [ "list", "diff" ], "type": "string", "description": "Choose list to dump every URL found right now, or diff to compare this run with the previous one and label each URL new, unchanged or removed. Diff mode needs at least two runs of the same source to be useful.", "default": "list" }, "diffStoreName": { "title": "Diff snapshot store name", "type": "string", "description": "Name of the named key-value store that keeps the URL snapshot between runs in diff mode, e.g. sitemap-monitor. Keep the same name across scheduled runs so they compare against each other; use different names to track the same site in separate pipelines.", "default": "sitemap-monitor" }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Enter how many sitemap files to download in parallel, e.g. 5. A sitemap index can point at hundreds of child files, so raise this to 10-20 for large sites and lower it to 1-2 if a server starts returning 429.", "default": 5 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }