{
"openapi": "3.0.1",
"info": {
"title": "Sitemap URL Extractor & Scraper: All Website URLs",
"description": "Sitemap scraper that extracts all URLs of a website from its XML sitemaps: sitemap.xml, sitemap indexes and .xml.gz files, found through robots.txt. Returns each URL with its last-modified date, and can check each URL's HTTP status to find dead pages and redirects. Export to CSV, Excel or JSON.",
"version": "1.0",
"x-build-id": "HrVxl200cDChHSH1P"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/cyber1nj3ct0r~sitemap-url-extractor/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-cyber1nj3ct0r-sitemap-url-extractor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/cyber1nj3ct0r~sitemap-url-extractor/runs": {
"post": {
"operationId": "runs-sync-cyber1nj3ct0r-sitemap-url-extractor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/cyber1nj3ct0r~sitemap-url-extractor/run-sync": {
"post": {
"operationId": "run-sync-cyber1nj3ct0r-sitemap-url-extractor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"required": [
"startUrls"
],
"properties": {
"startUrls": {
"title": "Websites or sitemaps",
"minItems": 1,
"type": "array",
"description": "Add website addresses (for example https://www.example.com) or sitemap files (for example https://www.example.com/sitemap.xml). For a website, the tool reads robots.txt to find the sitemaps and then tries the usual locations.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"url": {
"type": "string",
"title": "URL of a web page",
"format": "uri"
}
}
}
},
"maxUrls": {
"title": "Maximum URLs",
"minimum": 1,
"maximum": 5000000,
"type": "integer",
"description": "Stop after this many URLs. You pay per URL returned, so this also caps the cost of the run.",
"default": 10000
},
"checkStatus": {
"title": "Check that each URL works",
"type": "boolean",
"description": "Request every URL and add its HTTP status, redirect target and response time to the result. Use it to find dead pages (404) and redirects in a sitemap. Runs take longer.",
"default": false
},
"includePatterns": {
"title": "Only URLs matching",
"type": "array",
"description": "Keep a URL only if it matches at least one of these. Plain text matches anywhere in the URL and * stands for any characters, for example /blog/ or */products/*.html. For a regular expression, wrap it in slashes: /\\/p\\/\\d+$/.",
"items": {
"type": "string"
}
},
"excludePatterns": {
"title": "Skip URLs matching",
"type": "array",
"description": "Leave out a URL if it matches any of these. Same pattern rules as above, for example /tag/ or *.pdf.",
"items": {
"type": "string"
}
},
"modifiedAfter": {
"title": "Changed since",
"type": "string",
"description": "Keep only URLs whose last-modified date in the sitemap is on or after this date. Pick a date or a period such as 30 days. URLs without a date in the sitemap are left out when this filter is set."
},
"includeExtras": {
"title": "Include image and language entries",
"type": "boolean",
"description": "Add the image addresses and the language versions (hreflang) that the sitemap lists for each URL. Without this, the result only holds how many there are.",
"default": false
},
"requestTimeoutSecs": {
"title": "Time limit per request",
"minimum": 5,
"maximum": 120,
"type": "integer",
"description": "How long to wait for one file or page before giving up.",
"default": 30
},
"userAgent": {
"title": "User-Agent",
"type": "string",
"description": "The name the tool sends with each request. Leave empty to use the tool's own name, which tells site owners what is visiting. Set your own only for sites you run or have permission to test."
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}