{ "openapi": "3.0.1", "info": { "title": "Website URL Crawler & Link Extractor", "description": "Website URL Crawler: crawl any website and extract every URL with its depth, parent page and anchor text. Build a sitemap, map site structure or audit internal links for SEO. Works on static and JavaScript sites. Export to CSV, JSON, Excel or API.", "version": "1.0", "x-build-id": "ZT18Vb1XFW2XY4rTu" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/maged120~get-urls-pro/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-maged120-get-urls-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/maged120~get-urls-pro/runs": { "post": { "operationId": "runs-sync-maged120-get-urls-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/maged120~get-urls-pro/run-sync": { "post": { "operationId": "run-sync-maged120-get-urls-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrl" ], "properties": { "startUrl": { "title": "Start URL", "type": "string", "description": "The starting URL to crawl (e.g., https://jamesclear.com/five-step-creative-process)" }, "mode": { "title": "What to do", "enum": [ "crawl", "sitemap", "brokenLinks" ], "type": "string", "description": "Crawl is the classic mode (one row per URL found). Sitemap lists every URL in the site's sitemaps. Broken links crawls the site and checks every link. Leave empty to crawl." }, "renderJavaScript": { "title": "Render JavaScript", "type": "boolean", "description": "Turn on for sites built with React, Vue, Angular or other JavaScript frameworks. Slower; leave off for normal sites.", "default": false }, "allowDuplicates": { "title": "Allow Duplicate URLs", "type": "boolean", "description": "Allow duplicate URLs to be included in the output. If false, each URL will appear only once.", "default": false }, "maxDepth": { "title": "Maximum Crawl Depth", "minimum": 1, "maximum": 30, "type": "integer", "description": "Maximum depth of link recursion", "default": 3 }, "maxChildrenPerLink": { "title": "Maximum Children per Link", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum number of children links to follow from each parent link", "default": 3 }, "sameDomainOnly": { "title": "Same Domain Only", "type": "boolean", "description": "Only crawl URLs within the same domain as the start URL", "default": true }, "ignoredExtensions": { "title": "Ignored File Extensions", "uniqueItems": true, "type": "array", "description": "File extensions to ignore when crawling", "items": { "type": "string" } }, "websites": { "title": "More websites", "type": "array", "description": "Optional extra websites (example.com or full URLs) to process together with the start URL in the sitemap and broken links modes.", "items": { "type": "string" } }, "maxUrlsPerSite": { "title": "Max sitemap URLs per site", "minimum": 0, "type": "integer", "description": "Stop after this many page URLs per website. 0 = unlimited.", "default": 1000 }, "maxPagesPerSite": { "title": "Max pages to check per site", "minimum": 1, "type": "integer", "description": "Internal pages to visit per website. Every link on those pages is checked.", "default": 100 }, "checkExternalLinks": { "title": "Check external links", "type": "boolean", "description": "Also check links that point to other websites.", "default": true }, "checkImages": { "title": "Check images", "type": "boolean", "description": "Also check image sources for broken images.", "default": false }, "onlyBrokenLinks": { "title": "Output only broken links", "type": "boolean", "description": "Turn off to get every checked link with its status code.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }