{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & New Page Monitor", "description": "Get every URL from any website's XML sitemaps (robots.txt discovery, sitemap indexes, .gz, text, RSS/Atom) with lastmod, hreflang and images. Optional HTTP status check for 404s and redirects, and a monitor mode that returns only new pages. No browser, no start fee.", "version": "0.1", "x-build-id": "uBMhXAgOnwYFd23DK" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/open-data-tools~sitemap-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-open-data-tools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/open-data-tools~sitemap-extractor/runs": { "post": { "operationId": "runs-sync-open-data-tools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/open-data-tools~sitemap-extractor/run-sync": { "post": { "operationId": "run-sync-open-data-tools-sitemap-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemap URLs", "type": "array", "description": "One per line. A domain (example.com) or homepage finds the sitemaps through robots.txt and common locations. A direct sitemap URL (https://example.com/sitemap_index.xml, .xml.gz, .txt, RSS or Atom feed) is read as is.", "items": { "type": "string" } }, "maxUrlsPerSite": { "title": "Maximum URLs per site", "minimum": 1, "maximum": 2000000, "type": "integer", "description": "Stop reading a site's sitemaps after this many unique URLs. Filters below apply after this cap.", "default": 10000 }, "includePatterns": { "title": "Only URLs matching (regex)", "type": "array", "description": "Keep only URLs matching at least one of these regular expressions (case-insensitive). Example: /blog/ or \\.pdf$", "items": { "type": "string" } }, "excludePatterns": { "title": "Skip URLs matching (regex)", "type": "array", "description": "Drop URLs matching any of these regular expressions (case-insensitive). Example: /tag/ or \\?", "items": { "type": "string" } }, "lastmodFrom": { "title": "Modified on or after", "type": "string", "description": "Keep only URLs whose sitemap is on or after this date (YYYY-MM-DD). URLs without a lastmod are dropped when this is set." }, "onlyNewUrls": { "title": "Monitor: return only new URLs", "type": "boolean", "description": "Remembers every URL seen per site (in a named key-value store in your account). The first run saves a baseline and returns nothing, so it costs nothing per URL. Later runs return only URLs that were not in the sitemaps before. Schedule it daily to watch competitors' new pages, products or posts.", "default": false }, "stateStoreName": { "title": "Monitor state store name", "type": "string", "description": "Named key-value store that keeps the monitor's memory. Use different names for separate monitors.", "default": "sitemap-extractor-state" }, "checkStatus": { "title": "Check HTTP status of each URL", "type": "boolean", "description": "Sends a HEAD request (GET if HEAD is refused) to every output URL and records the status code, redirect and final URL. Finds 404s and redirects listed in sitemaps. Charged as a separate event per URL.", "default": false }, "statusCheckConcurrency": { "title": "Status check concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Parallel status requests. Keep it low for small sites.", "default": 10 }, "maxSitemapsPerSite": { "title": "Maximum sitemap files per site", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Safety cap for huge sitemap indexes.", "default": 500 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 300, "type": "integer", "description": "How long to wait for one sitemap or status request before retrying.", "default": 30 }, "userAgent": { "title": "User-Agent", "type": "string", "description": "Optional. By default the actor identifies itself as SitemapExtractor. Set your own if you are checking your own site behind a firewall rule." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }