{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & Sitemap Scraper - All Website URLs", "description": "Get every URL of a website from its sitemaps: finds them via robots.txt, follows indexes and .gz files, returns lastmod, images and hreflang, optional status codes and changes. $0.50/1k URLs.", "version": "0.1", "x-build-id": "qRU6Y6I7UIizPwbxp" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/tidytools~sitemap-url-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-tidytools-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/tidytools~sitemap-url-extractor/runs": { "post": { "operationId": "runs-sync-tidytools-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/tidytools~sitemap-url-extractor/run-sync": { "post": { "operationId": "run-sync-tidytools-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "urls": { "title": "Websites or sitemap URLs", "type": "array", "description": "Main input (fill this, or `startUrls`). One website (e.g. example.com or https://example.com — sitemaps are found via robots.txt and common locations) or direct sitemap URL (e.g. https://example.com/sitemap.xml) per line. Bare domains are fine; blank or invalid lines are skipped and listed in the results (not charged).", "items": { "type": "string" } }, "maxUrls": { "title": "Max URLs", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Stop after this many URLs in total (all websites together). You pay only for URLs returned: 10,000 URLs cost at most $5 on the Free plan and $1.50 on Business. Raise it for bigger sites.", "default": 10000 }, "maxUrlsPerSite": { "title": "Max URLs per website", "minimum": 0, "maximum": 1000000, "type": "integer", "description": "With several websites: stop taking URLs from one website after this many, so every website gets its share of Max URLs. 0 = no per-website limit.", "default": 0 }, "startUrls": { "title": "Websites or sitemap URLs (request list, optional)", "type": "array", "description": "Same as the list above, in Apify's request-list format (also accepts a link to a text file of URLs). Kept for older inputs; the plain list above is recommended, especially for API calls.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "includeUrlPatterns": { "title": "Include URL patterns", "type": "array", "description": "Optional glob patterns; only matching URLs are returned, e.g. https://example.com/blog/**", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns", "type": "array", "description": "Optional glob patterns to drop, e.g. **/tag/**", "items": { "type": "string" } }, "modifiedAfter": { "title": "Only URLs modified after", "type": "string", "description": "Optional date (YYYY-MM-DD). Keeps URLs whose lastmod is on or after this date; for Google News entries without lastmod, a valid publication date is used instead. URLs without either date are kept." }, "urlsOnly": { "title": "Output URLs only", "type": "boolean", "description": "Return one short row per URL with only the url column (plus changeType when comparison is on and httpStatus when status checks are on), for spreadsheets, n8n or Make. Same price; error rows for websites without a sitemap are kept as they are.", "default": false }, "checkStatus": { "title": "Check each URL's HTTP status", "type": "boolean", "description": "Request every extracted URL (HEAD, falling back to GET) and add httpStatus, statusClass (ok, redirect, broken, restricted, server-error, error), finalUrl and redirected. Charged $0.30 per 1,000 checked URLs on top of the URL price; URLs that could not be reached at all are not charged.", "default": false }, "compareWithPrevious": { "title": "Compare with the previous run", "type": "boolean", "description": "Remember this run's URLs and lastmod dates, and mark each URL as new, updated or unchanged compared with the last run of the same monitor. URLs that disappeared are added as \"removed\". Ideal for scheduled runs.", "default": false }, "onlyChanges": { "title": "Only output changes", "type": "boolean", "description": "With comparison on, output only new, updated and removed URLs (unchanged URLs are not output and not charged).", "default": false }, "stateKey": { "title": "Monitor name", "type": "string", "description": "Name for the saved state (letters, digits, dashes). Runs with the same name are compared with each other. Default: the first start URL's host name. Use a new name if you change the URL filters." }, "maxConcurrency": { "title": "Parallel status checks", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many URLs are status-checked at the same time.", "default": 20 }, "siteTimeoutSecs": { "title": "Time limit per website (seconds)", "minimum": 0, "maximum": 3600, "type": "integer", "description": "Reading one website (robots.txt, all its sitemap files and their status checks) stops after this long. URLs found before the limit stay in the results (charged); the website gets a row with errorType \"timeout\" (not charged) and the next website starts. 0 = no limit.", "default": 300 }, "maxConcurrentSites": { "title": "Parallel websites", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many websites are read at the same time, so one slow website does not hold up the others. Each website keeps its own time limit. Parallel status checks are shared by all websites. With several websites in parallel, Max URLs goes to whichever websites list URLs first: set Max URLs per website to share it evenly.", "default": 5 }, "proxyConfiguration": { "title": "Proxy (optional)", "type": "object", "description": "Optional. Used for sitemap downloads and status checks (all requests come from Apify's network). Apify proxy usage is billed to your Apify account." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }