{ "openapi": "3.0.1", "info": { "title": "URL & Sitemap Link Extractor: Site Map, Health Checks & Monitor", "description": "Extract every URL from a site's sitemap or a page, no sitemap link needed. Auto-discovers via robots.txt, follows sitemap-index nesting, keeps lastmod, changefreq, and priority. Adds a site-structure summary, link-health checks, and change monitoring. Export as JSON, CSV, or Excel. $0.00001 per URL.", "version": "0.1", "x-build-id": "0POxGXtqbje0Fm3Kv" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/getascraper~url-link-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-getascraper-url-link-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/getascraper~url-link-extractor/runs": { "post": { "operationId": "runs-sync-getascraper-url-link-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/getascraper~url-link-extractor/run-sync": { "post": { "operationId": "run-sync-getascraper-url-link-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "domain": { "title": "Domain", "type": "string", "description": "A bare domain (e.g. example.com). The Actor checks robots.txt for its declared sitemap(s) and follows any sitemap index automatically, no need to know the exact sitemap URL. Ignored whenever Specific Sitemap or Page URLs below is filled in, so clear this field or add your own URLs there, not both.", "default": "apify.com" }, "links": { "title": "Specific Sitemap or Page URLs", "type": "array", "description": "One or more exact sitemap URLs or webpage URLs to extract from, when you already know them. A sitemap index is still followed automatically. A plain webpage URL returns the links found on that one page (no further crawling).", "default": [], "items": { "type": "string" } }, "urlPattern": { "title": "Include Path Pattern", "type": "string", "description": "Only keep URLs whose path contains one of these comma-separated patterns (e.g. \"/blog, /products\"). Leave blank to keep every URL.", "default": "" }, "customExcludePattern": { "title": "Exclude Path Pattern", "type": "string", "description": "Drop URLs whose path contains any of these comma-separated patterns (e.g. \"/tag, /author\"). Leave blank to skip this filter.", "default": "" }, "excludeExtensions": { "title": "Exclude File Extensions", "type": "string", "description": "Drop URLs ending in these comma-separated file extensions (e.g. \"pdf, jpg, zip\"). Leave blank to skip this filter.", "default": "" }, "checkLinkHealth": { "title": "Check Link Health", "type": "boolean", "description": "For a bounded sample of the returned URLs, send a request to each and record its HTTP status, so you can spot broken links. Costs one extra request per checked URL, so it defaults off.", "default": false }, "maxUrls": { "title": "Max URLs", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Maximum number of URLs to return in this run, after filters are applied.", "default": 25 }, "maxSitemapsToFollow": { "title": "Max Sitemaps to Follow", "minimum": 1, "maximum": 200, "type": "integer", "description": "Maximum number of sitemap or page URLs to actually fetch. A sitemap index can list hundreds or thousands of nested sitemaps (a real large site can have 2,000+); this caps how many of those are followed in one run.", "default": 5 }, "onlyNewOrChanged": { "title": "Incremental Monitor Mode", "type": "boolean", "description": "Only return URLs that are new or whose lastmod/changefreq/priority changed since your last run with the same monitor state name. Pair with an Apify schedule to catch new or updated pages without re-downloading the whole sitemap every time.", "default": false }, "stateName": { "title": "Monitor State Name", "type": "string", "description": "Name for this run's incremental state, used only when Incremental Monitor Mode is on. Use different names to track separate domains or filters independently.", "default": "default" }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "This Actor makes plain HTTP requests to fetch sitemaps and pages. Datacenter proxy (the default) is fast and works for every target checked during development; switch to Residential only if a specific site you're targeting starts blocking datacenter IPs.", "default": { "useApifyProxy": true } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }