{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor: robots.txt, gzip, Changes", "description": "Extract every URL from a website's XML sitemaps: found through robots.txt, sitemap indexes and .gz files followed, lastmod and date filters, URL patterns, and new, changed or removed URLs since your last run. Reads sitemap files only, never the pages.", "version": "0.2", "x-build-id": "zHYnzbdSl9Cgo82aJ" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/succinct_glider~sitemap-urls/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-succinct_glider-sitemap-urls", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/succinct_glider~sitemap-urls/runs": { "post": { "operationId": "runs-sync-succinct_glider-sitemap-urls", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/succinct_glider~sitemap-urls/run-sync": { "post": { "operationId": "run-sync-succinct_glider-sitemap-urls", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemaps", "type": "array", "description": "One entry per line: a domain (example.com), a page of the site (https://example.com/blog), or the URL of a sitemap or sitemap index (https://example.com/sitemap.xml, .xml.gz). For a domain or page, the sitemaps listed in robots.txt are used, then the usual locations such as /sitemap.xml.", "items": { "type": "string" } }, "maxItems": { "title": "Max URLs", "minimum": 1, "maximum": 5000000, "type": "integer", "description": "Maximum number of URLs saved in this run, across all websites. The run stops reading sitemaps as soon as it is reached.", "default": 500 }, "maxItemsPerSite": { "title": "Max URLs per website", "minimum": 0, "type": "integer", "description": "Limit per website, so one large site cannot use the whole budget. 0 means no limit.", "default": 0 }, "includeUrlPatterns": { "title": "Only URLs matching", "type": "array", "description": "Keep only URLs that match one of these patterns. `*` matches any characters, e.g. `*/blog/*` or `https://example.com/products/*`; a pattern without `*` matches URLs that contain it. Matching is case-insensitive. Filtering happens before saving, so filtered-out URLs are never charged.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Skip URLs matching", "type": "array", "description": "Skip URLs that match one of these patterns, e.g. `*/tag/*` or `*?page=*`. Applied after the include patterns.", "items": { "type": "string" } }, "modifiedSince": { "title": "Modified since", "type": "string", "description": "Keep only URLs whose sitemap `lastmod` is on or after this date. Absolute (2026-09-01) or relative (7 days). URLs without a `lastmod` are kept unless you enable the option below." }, "requireLastmod": { "title": "Skip URLs without a lastmod date", "type": "boolean", "description": "With a date filter, drop URLs whose sitemap entry has no `lastmod`. Many sites never set it.", "default": false }, "sitemapUrlPatterns": { "title": "Only read sitemaps matching", "type": "array", "description": "On large sites, follow only the child sitemaps of an index whose URL matches one of these patterns, e.g. `*blog*` or `*products*`. The sitemaps you list yourself are always read.", "items": { "type": "string" } }, "trackChanges": { "title": "Track changes", "type": "boolean", "description": "Compare each website with its previous tracked run and add `changeStatus` (new, changed, unchanged, removed). Removed URLs are reported only when the website was read completely in both runs.", "default": false }, "onlyChanges": { "title": "Only save new, changed and removed URLs", "type": "boolean", "description": "Skip unchanged URLs: they are not saved, so you are not charged for them. Enables change tracking automatically. The first run saves everything.", "default": false }, "stateStoreName": { "title": "State store name", "pattern": "^[a-zA-Z0-9-]{1,63}$", "type": "string", "description": "Named key-value store of your account that keeps the previous state of each website. Use a different name for each independent list you track. Letters, digits and hyphens only.", "default": "sitemap-urls-state" }, "checkUrlStatus": { "title": "Check the HTTP status of each URL", "type": "boolean", "description": "Send a HEAD request to every saved URL and add its HTTP status (`statusCode`) and redirect target (`redirectUrl`), without following redirects. Slower, and charged as an extra event per checked URL (see the Pricing tab). Off by default: most uses only need the list.", "default": false }, "includeAlternates": { "title": "Include language alternates", "type": "boolean", "description": "Add the `hreflang` alternate URLs declared for each entry (`xhtml:link`). They can make items much larger on multilingual sites.", "default": false }, "useDefaultLocations": { "title": "Try the usual sitemap locations", "type": "boolean", "description": "When robots.txt lists no sitemap, also try /sitemap.xml, /sitemap_index.xml and /sitemap-index.xml.", "default": true }, "maxSitemapsPerSite": { "title": "Max sitemap files per website", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Safety limit on the number of sitemap files read for one website, including nested indexes.", "default": 1000 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Sitemaps are public files meant for crawlers and usually need no proxy.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }