{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & New Page Monitor", "description": "Get every URL from any website's sitemap.xml (auto-discovered via robots.txt, .gz and nested indexes) with lastmod, hreflang, image and news data. Filter by date or regex, check status codes, or monitor competitors for new pages. Fast, no browser, no proxy.", "version": "1.0", "x-build-id": "9psDKxbEkrOc7VjKY" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/thomson-digital~sitemap-url-monitor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-thomson-digital-sitemap-url-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/thomson-digital~sitemap-url-monitor/runs": { "post": { "operationId": "runs-sync-thomson-digital-sitemap-url-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/thomson-digital~sitemap-url-monitor/run-sync": { "post": { "operationId": "run-sync-thomson-digital-sitemap-url-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites, robots.txt or sitemap URLs", "type": "array", "description": "One per line. Enter a domain (example.com), a homepage URL, a robots.txt URL, or a direct sitemap URL (.xml, .xml.gz or .txt). For domains, sitemaps are discovered from robots.txt and common locations such as /sitemap.xml, /sitemap_index.xml and /wp-sitemap.xml.", "default": [ "https://docs.apify.com" ], "items": { "type": "string" } }, "maxUrlsPerSite": { "title": "Max URLs per site", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Maximum number of URLs to output for each website. Keep it small for testing. Sitemap download stops early once the limit is reached (except in monitor mode, which must read the whole sitemap to detect new URLs).", "default": 100 }, "includeUrlPatterns": { "title": "Only include URLs matching (regex)", "type": "array", "description": "Optional. Keep only URLs matching at least one of these regular expressions (case-insensitive), e.g. /blog/ or /products/.", "default": [], "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URLs matching (regex)", "type": "array", "description": "Optional. Drop URLs matching any of these regular expressions, e.g. /tag/ or \\?page=.", "default": [], "items": { "type": "string" } }, "modifiedWithinDays": { "title": "Only URLs modified in the last N days", "minimum": 0, "maximum": 36500, "type": "integer", "description": "Optional. Keep only URLs whose sitemap is within the last N days. Child sitemaps with an older are skipped entirely, which makes runs on big sites much faster and cheaper." }, "modifiedAfter": { "title": "Only URLs modified after date", "type": "string", "description": "Optional. ISO date such as 2026-01-31. Same effect as the option above, using a fixed date." }, "keepUrlsWithoutLastmod": { "title": "Keep URLs without when filtering by date", "type": "boolean", "description": "Many sitemaps omit . When a date filter is set, such URLs are dropped unless you enable this.", "default": false }, "onlyNewUrls": { "title": "Monitor mode: output only new URLs", "type": "boolean", "description": "Remembers every URL seen per site (in your private named key-value store 'sitemap-url-monitor-state') and outputs only URLs that were not present in previous runs. Schedule the Actor daily or weekly to get alerts about new pages, products or articles. The first run only saves a baseline.", "default": false }, "emitAllOnFirstRun": { "title": "Output all URLs on the first monitor run", "type": "boolean", "description": "In monitor mode, the first run normally outputs nothing and just saves the baseline. Enable this to output all URLs on that first run (marked isNew=true).", "default": false }, "checkStatus": { "title": "Check HTTP status of each URL", "type": "boolean", "description": "Sends a lightweight HEAD request to every output URL and adds statusCode, finalUrl and redirected. Useful for SEO audits (find 404s and redirects listed in sitemaps). Slower.", "default": false }, "maxSitemapsPerSite": { "title": "Max sitemap files per site", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Safety cap on the number of sitemap files downloaded per website.", "default": 200 }, "maxDepth": { "title": "Max sitemap index depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many levels of nested sitemap indexes to follow.", "default": 5 }, "maxConcurrency": { "title": "Max parallel requests per site", "minimum": 1, "maximum": 20, "type": "integer", "description": "Number of sitemap files downloaded in parallel for each site. Be polite.", "default": 5 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 180, "type": "integer", "description": "Timeout for each HTTP request.", "default": 30 }, "maxRetries": { "title": "Max retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Retries for network errors, HTTP 429 and 5xx responses (exponential backoff, honors Retry-After).", "default": 3 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Not needed for most sites. Use Apify Proxy only if a site blocks plain requests to its sitemap.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }