{ "openapi": "3.0.1", "info": { "title": "Sitemap Scraper - Find the Sitemap and Watch for New URLs", "description": "For SEO audits, crawl seeding and keeping a RAG index fresh: give it a domain and get one row per URL with lastmod, change frequency and priority. Asking for /sitemap.xml fails on real sites - stripe.com answers 404 - so it reads robots.txt, follows nested indexes and unpacks gzip. No API key.", "version": "0.1", "x-build-id": "Wz9jvfUQzSNKShqOm" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/neverempty~sitemap-finder-monitor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-neverempty-sitemap-finder-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/neverempty~sitemap-finder-monitor/runs": { "post": { "operationId": "runs-sync-neverempty-sitemap-finder-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/neverempty~sitemap-finder-monitor/run-sync": { "post": { "operationId": "run-sync-neverempty-sitemap-finder-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "sites": { "title": "Sites", "type": "array", "description": "Domains or URLs. A bare domain such as stripe.com is enough: the Actor reads robots.txt to find where that site actually keeps its sitemap, and only falls back to guessing common locations if robots.txt declares none. If this, the paste box and the sitemap URLs are all left empty, the example sites stripe.com and vercel.com are read and a free row says so. If the two example sites are still here when you give your own sites in the paste box or sitemap URLs, they are skipped and not charged.", "items": { "type": "string" } }, "sitesText": { "title": "Sites (paste, one per line)", "type": "string", "description": "The same list as a block you can paste. Both fields are merged and duplicates are dropped.", "default": "" }, "sitemapUrls": { "title": "Sitemap URLs (skip the search)", "type": "array", "description": "Give sitemap files directly when you already know them. These are read in addition to whatever the sites above turn up.", "default": [], "items": { "type": "string" } }, "useCommonPaths": { "title": "Try common locations when robots.txt declares none", "type": "boolean", "description": "Tries /sitemap.xml, /sitemap_index.xml, /sitemap/sitemap.xml and six more. Turn this off to report only what a site officially declares.", "default": true }, "maxUrls": { "title": "Maximum URLs", "minimum": 1, "maximum": 200000, "type": "integer", "description": "Stop after this many URLs across all sites in this run.", "default": 1000 }, "maxUrlsPerSite": { "title": "Maximum URLs per site", "minimum": 0, "maximum": 200000, "type": "integer", "description": "Cap each site separately so one large site cannot use up the whole run. 0 = no per-site cap.", "default": 0 }, "maxSitemaps": { "title": "Maximum sitemap files", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Large sites split their sitemap into many files. This caps how many files are opened.", "default": 50 }, "maxSitemapDepth": { "title": "Maximum index depth", "minimum": 1, "maximum": 6, "type": "integer", "description": "A sitemap index can point at other indexes. This is how many levels down to follow.", "default": 3 }, "timeoutSecs": { "title": "Timeout per request (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Give up on a single sitemap file after this long. The other files are still returned.", "default": 20 }, "pathIncludes": { "title": "Path contains", "type": "array", "description": "Keep only URLs whose address contains one of these, for example /blog/ or /docs/. Empty keeps everything.", "default": [], "items": { "type": "string" } }, "pathExcludes": { "title": "Path does not contain", "type": "array", "description": "Drop URLs whose address contains any of these, for example /tag/ or ?page=.", "default": [], "items": { "type": "string" } }, "extensions": { "title": "File extensions", "type": "array", "description": "Keep only URLs ending in these, for example pdf or html. A URL with no extension is dropped when this is set.", "default": [], "items": { "type": "string" } }, "lastmodAfter": { "title": "Changed on or after (YYYY-MM-DD)", "type": "string", "description": "Keep URLs the sitemap says were changed on or after this date. A URL with no date is kept, because unknown is not the same as old.", "default": "" }, "lastmodBefore": { "title": "Changed on or before (YYYY-MM-DD)", "type": "string", "description": "Keep URLs the sitemap says were changed on or before this date. A URL with no date is kept.", "default": "" }, "minPriority": { "title": "Minimum priority", "minimum": 0, "maximum": 1, "type": "integer", "description": "Sitemaps may carry a priority from 0 to 1. This keeps only URLs at or above it. A URL with no priority is kept.", "default": 0 }, "changefreqIn": { "title": "Change frequency", "type": "array", "description": "Keep only URLs the sitemap marks with these frequencies, such as daily or weekly.", "default": [], "items": { "type": "string" } }, "keywords": { "title": "Keywords", "type": "array", "description": "Keep only URLs containing these words. Matched against the whole address.", "default": [], "items": { "type": "string" } }, "keywordMatch": { "title": "Keyword match", "enum": [ "any", "all" ], "type": "string", "description": "any = contains at least one. all = contains every one.", "default": "any" }, "excludeKeywords": { "title": "Exclude keywords", "type": "array", "description": "Drop URLs containing any of these words.", "default": [], "items": { "type": "string" } }, "monitoringMode": { "title": "Monitoring mode: return only URLs that are new", "type": "boolean", "description": "Reads the same sites again and returns only the URLs that were not there last time. The first run returns everything and remembers it. Nothing new means nothing returned and nothing charged, so watching a competitor for new pages costs nothing on quiet days.", "default": false }, "resetMonitoringState": { "title": "Forget what was already returned", "type": "boolean", "description": "Clears the memory so the next monitoring run treats every URL as new.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }