{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor – All URLs from sitemap.xml & robots.txt", "description": "Extract page URLs from a website's sitemaps. Finds sitemaps via robots.txt, common paths and homepage links, then reads XML and .xml.gz sitemaps, sitemap indexes, text sitemaps, RSS and Atom feeds. Returns lastmod, changefreq, priority, images, videos and hreflang.", "version": "1.0", "x-build-id": "p5zhpdHJd08jAiO8U" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/openharvest~sitemap-url-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-openharvest-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/openharvest~sitemap-url-extractor/runs": { "post": { "operationId": "runs-sync-openharvest-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/openharvest~sitemap-url-extractor/run-sync": { "post": { "operationId": "run-sync-openharvest-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemap URLs", "type": "array", "description": "Domains (https://example.com or example.com) or direct links to sitemap.xml, sitemap index, .xml.gz, .txt sitemaps, RSS or Atom feeds.", "default": [ { "url": "https://apify.com" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxUrls": { "title": "Max URLs", "minimum": 0, "type": "integer", "description": "Maximum number of URLs to return in total (0 = unlimited). You only pay for URLs actually returned.", "default": 500 }, "maxUrlsPerSite": { "title": "Max URLs per website", "minimum": 0, "type": "integer", "description": "Optional cap per start URL so one huge website does not use the whole Max URLs budget when you process several sites (0 = no per-site cap).", "default": 0 }, "followSitemapIndex": { "title": "Follow sitemap indexes", "type": "boolean", "description": "Open child sitemaps listed in sitemap index files (recommended).", "default": true }, "includeRobotsTxtDiscovery": { "title": "Discover sitemaps from robots.txt", "type": "boolean", "description": "Read the site's robots.txt and use every Sitemap: line.", "default": true }, "tryCommonPaths": { "title": "Also try common sitemap paths", "type": "boolean", "description": "Probe /sitemap.xml, /sitemap_index.xml, /sitemap-index.xml, /wp-sitemap.xml, /sitemap.xml.gz, /news-sitemap.xml, /sitemap_news.xml and /sitemap.txt. Missing ones are skipped silently.", "default": true }, "discoverFromHomepage": { "title": "Discover sitemaps & RSS/Atom feeds from the homepage", "type": "boolean", "description": "If robots.txt lists no sitemap, look for and RSS/Atom feed links on the homepage.", "default": true }, "urlIncludeRegex": { "title": "Include URLs matching (regex)", "type": "string", "description": "Only return URLs matching this regular expression (case-insensitive). Example: /blog/ or \\.html$" }, "urlExcludeRegex": { "title": "Exclude URLs matching (regex)", "type": "string", "description": "Skip URLs matching this regular expression (case-insensitive). Example: /tag/|/author/" }, "sameDomainOnly": { "title": "Only URLs on the same domain", "type": "boolean", "description": "Drop URLs whose host differs from the start website (subdomains are kept).", "default": false }, "includeMeta": { "title": "Include metadata", "type": "boolean", "description": "Add lastmod, changefreq, priority, image/video/news extensions, hreflang alternates and feed titles. Turn off for a lean URL-only list.", "default": true }, "dedupe": { "title": "Remove duplicate URLs", "type": "boolean", "description": "Return each URL only once even if it appears in several sitemaps (duplicates are never charged).", "default": true }, "saveSummary": { "title": "Save run summary (OUTPUT record)", "type": "boolean", "description": "Store a summary with URL counts per sitemap, discovery source and errors in the key-value store under OUTPUT.", "default": true }, "maxConcurrency": { "title": "Max parallel downloads", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many sitemaps are downloaded in parallel.", "default": 8 }, "maxSitemaps": { "title": "Max sitemaps per website", "minimum": 1, "type": "integer", "description": "Safety cap on the number of sitemap files opened for each start URL (huge sites can have thousands of child sitemaps).", "default": 10000 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 300, "type": "integer", "description": "Timeout for the first byte and for idle connections. Large sitemaps keep streaming as long as data flows.", "default": 30 }, "maxRetries": { "title": "Max retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Retries for network errors, timeouts, 429 and 5xx responses (exponential backoff).", "default": 3 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Most sitemaps are public and need no proxy. Enable Apify Proxy for sites that block data-center IPs.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }