{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor + Change Monitor (robots.txt, gzip)", "description": "Every URL from a site's sitemaps: robots.txt discovery, sitemap index recursion, gzip, lastmod filter, image/video/news/hreflang fields. Or run it on a schedule and get only the URLs added, removed or changed since last time.", "version": "0.1", "x-build-id": "judefNpEinkYWXT4Z" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/osel_house~sitemap-extractor-monitor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-osel_house-sitemap-extractor-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/osel_house~sitemap-extractor-monitor/runs": { "post": { "operationId": "runs-sync-osel_house-sitemap-extractor-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/osel_house~sitemap-extractor-monitor/run-sync": { "post": { "operationId": "run-sync-osel_house-sitemap-extractor-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemaps", "type": "array", "description": "A domain or homepage (sitemaps are found through robots.txt and the usual locations), a robots.txt URL, or a sitemap URL (.xml, .xml.gz, sitemap index, .txt, RSS/Atom). One entry per site.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "mode": { "title": "What to output", "enum": [ "all", "changes" ], "type": "string", "description": "All URLs = every URL in the sitemaps. Only changes = compare with the previous run for the same sites and output only URLs that were added, removed or had their lastmod changed (the first run stores a baseline).", "default": "all" }, "maxUrls": { "title": "Max URLs", "minimum": 1, "type": "integer", "description": "Stop after this many URLs (after filters). The run says so when the cap is hit.", "default": 500000 }, "lastmodAfter": { "title": "Only URLs modified after", "type": "string", "description": "Keep only URLs whose is after this point. A date (2026-09-01), a datetime (2026-09-01T00:00:00Z) or a span back from now: 7d, 36h, 2 weeks. Great for scheduled runs that feed a scraper only fresh pages." }, "includeUrlsWithoutLastmod": { "title": "Keep URLs that have no lastmod", "type": "boolean", "description": "Only matters with \"Only URLs modified after\". Off: URLs without a are dropped (counted in the summary). On: they are kept, since they might be new.", "default": false }, "urlIncludePatterns": { "title": "URL must match (regex)", "type": "array", "description": "Keep only URLs matching at least one of these regular expressions, e.g. /blog/ or \\.html$. Leave empty to keep all.", "items": { "type": "string" } }, "urlExcludePatterns": { "title": "URL must not match (regex)", "type": "array", "description": "Drop URLs matching any of these regular expressions, e.g. \\?page= or /tag/.", "items": { "type": "string" } }, "dedupeUrls": { "title": "Remove duplicate URLs", "type": "boolean", "description": "The same URL listed in several sitemaps is output once (first occurrence wins). Duplicates are counted in the summary.", "default": true }, "includeExtensions": { "title": "Include image, video, news and hreflang data", "type": "boolean", "description": "Extract sitemap extensions into the images, videos, news and alternates fields. Turn off for a leaner dataset.", "default": true }, "snapshotKey": { "title": "Snapshot name", "type": "string", "description": "Name of the saved URL set to compare against. Leave empty to derive it from the start URLs and patterns (recommended). Set it when you want two different inputs to share one baseline, or to keep several baselines for one site." }, "firstRunOutput": { "title": "First run", "enum": [ "baseline-only", "all-as-added" ], "type": "string", "description": "What the first run (no snapshot yet) outputs. Baseline only = store the URL set, output nothing. All as added = also output every URL with change = added.", "default": "baseline-only" }, "allowMassRemoval": { "title": "Allow mass removal", "type": "boolean", "description": "Report removals even when more than half of the baseline disappeared in one run (a site restructure). Off by default so a broken sitemap never floods you with false removals.", "default": false }, "resetSnapshot": { "title": "Reset the snapshot", "type": "boolean", "description": "Delete the saved baseline first; this run stores a new one. Turn it off again afterwards.", "default": false }, "checkStatus": { "title": "Check page status", "type": "boolean", "description": "Request each URL (the transfer is cut off as soon as the headers arrive) and record the HTTP status and final URL after redirects. Off by default: it is one request per URL and slows large sites down.", "default": false }, "maxStatusChecks": { "title": "Max status checks per run", "minimum": 1, "type": "integer", "description": "Safety cap; rows beyond it get status null.", "default": 5000 }, "aiBotSummary": { "title": "AI crawler policy summary", "type": "boolean", "description": "Report in the run summary whether robots.txt blocks GPTBot, ClaudeBot, PerplexityBot, Google-Extended, CCBot and other AI crawlers, and whether /llms.txt exists.", "default": true }, "maxSitemaps": { "title": "Max sitemap files", "minimum": 1, "type": "integer", "description": "Cap on sitemap files fetched per run (indexes can list thousands).", "default": 2000 }, "maxDepth": { "title": "Max index nesting", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many levels of sitemap indexes to follow (0 = do not follow indexes).", "default": 5 }, "maxSitemapMegabytes": { "title": "Max size per sitemap (MB, uncompressed)", "minimum": 1, "maximum": 2000, "type": "integer", "description": "A sitemap larger than this is read up to the cap and marked truncated. The protocol limit is 50 MB.", "default": 100 }, "maxConcurrency": { "title": "Parallel sitemap requests", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many sitemap files to fetch at the same time.", "default": 5 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 300, "type": "integer", "description": "Per request. Retries with backoff happen on 429/5xx and network errors.", "default": 30 }, "customUserAgent": { "title": "User-Agent", "type": "string", "description": "Leave empty for a real browser header set. Set it to identify your crawler (some sites whitelist known agents)." }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Use Apify Proxy when a site blocks datacenter requests (403 on robots.txt or sitemap). Off by default; most sitemaps do not need it." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }