{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor — PDF/DOCX Tags + robots.txt", "description": "Discover every URL in XML sitemaps via robots.txt and indexes; tag PDF, DOCX, XLSX and more for SEO audits and RAG ingest. Auto-finds nested indexes and .xml.gz; exports DOC_TO_MARKDOWN_INPUT for the PDF/DOCX Actor.", "version": "0.1", "x-build-id": "aav4teVcM8KNiuiQ4" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/ingenious_quip_bxq~sitemap-url-discovery/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-ingenious_quip_bxq-sitemap-url-discovery", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/ingenious_quip_bxq~sitemap-url-discovery/runs": { "post": { "operationId": "runs-sync-ingenious_quip_bxq-sitemap-url-discovery", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/ingenious_quip_bxq~sitemap-url-discovery/run-sync": { "post": { "operationId": "run-sync-ingenious_quip_bxq-sitemap-url-discovery", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or sitemap URLs", "type": "array", "description": "A homepage like `https://www.example.com` (sitemaps are discovered from robots.txt and /sitemap.xml, /sitemap_index.xml) or a direct sitemap URL (`.xml`, `.xml.gz`, `.txt`, sitemap index).", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxUrls": { "title": "Max URLs", "minimum": 0, "maximum": 1000000, "type": "integer", "description": "Stop after saving this many unique URLs (0 = no limit). Also caps your cost: you pay only for URLs saved.", "default": 1000 }, "maxUrlsPerInput": { "title": "Max URLs per website", "minimum": 0, "type": "integer", "description": "Optional cap per input website/sitemap (0 = no cap), so one large site cannot use up the whole Max URLs budget.", "default": 0 }, "fileTypes": { "title": "Only these file types", "type": "array", "description": "Keep only URLs with these file-type tags (by extension). Leave empty to keep everything. Example: only `pdf` and `docx` to list a site's documents.", "items": { "type": "string", "enum": [ "pdf", "docx", "doc", "xlsx", "xls", "pptx", "ppt", "csv", "html", "image", "other" ], "enumTitles": [ "PDF", "Word DOCX", "Word DOC (legacy)", "Excel XLSX", "Excel XLS (legacy)", "PowerPoint PPTX", "PowerPoint PPT (legacy)", "CSV", "HTML page", "Image", "Other" ] } }, "lastmodFrom": { "title": "Changed on or after", "type": "string", "description": "Keep only URLs whose sitemap `lastmod` is on or after this date (YYYY-MM-DD). Useful for monitoring new or updated pages." }, "includeUrlsWithoutLastmod": { "title": "Keep URLs without lastmod", "type": "boolean", "description": "When a date filter is set, keep URLs that have no `lastmod` in the sitemap.", "default": true }, "includeUrlPatterns": { "title": "Include URL patterns (regex)", "type": "array", "description": "Keep only URLs matching at least one of these regular expressions, e.g. `/blog/`.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns (regex)", "type": "array", "description": "Drop URLs matching any of these regular expressions, e.g. `/tag/`.", "items": { "type": "string" } }, "sameDomainOnly": { "title": "Same domain only", "type": "boolean", "description": "Drop URLs whose host differs from the input website (www. ignored). Off by default because many sites list files on a CDN host.", "default": false }, "checkContentType": { "title": "Confirm file type with HTTP HEAD", "enum": [ "off", "documents", "all" ], "type": "string", "description": "Send a HEAD request to confirm HTTP status and Content-Type. `documents` checks only non-HTML, non-image URLs (e.g. files and URLs without a clear extension); `all` checks every URL. Costs an extra event per URL checked and makes runs slower.", "default": "off" }, "useRobotsTxt": { "title": "Read robots.txt", "type": "boolean", "description": "Discover sitemaps from `Sitemap:` lines in robots.txt.", "default": true }, "tryCommonPaths": { "title": "Try common sitemap paths", "type": "boolean", "description": "Also try /sitemap.xml and /sitemap_index.xml. A missing path is reported as `not_found`, not as an error.", "default": true }, "maxDepth": { "title": "Max sitemap-index depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many levels of nested sitemap indexes to follow.", "default": 5 }, "maxSitemaps": { "title": "Max sitemaps", "minimum": 0, "type": "integer", "description": "Maximum number of sitemap files to read per input website (0 = no limit).", "default": 500 }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Parallel requests overall.", "default": 4 }, "maxConcurrencyPerHost": { "title": "Max concurrency per host", "minimum": 1, "maximum": 10, "type": "integer", "description": "Parallel requests to any single host (kept low to be polite).", "default": 2 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 180, "type": "integer", "description": "Timeout for each HTTP request.", "default": 30 }, "maxRetries": { "title": "Max retries", "minimum": 0, "maximum": 8, "type": "integer", "description": "Retries for timeouts, network errors, HTTP 429 and 5xx (exponential backoff, honours Retry-After up to 30 s).", "default": 3 }, "userAgent": { "title": "User-Agent", "type": "string", "description": "Optional custom User-Agent header." }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional. Use a proxy if a site blocks data-center IPs. Not needed for most sites.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }