{ "openapi": "3.0.1", "info": { "title": "Website URL Crawler, Link Extractor & Sitemap Audit", "description": "Crawl public websites and extract URLs, parent links, anchor text, depth, status codes, redirects, canonicals, indexability, and sitemap gaps. Static and JavaScript pages. $0.005 per useful URL; failed and diagnostic rows are free.", "version": "0.1", "x-build-id": "gaxD2hHdQRuotLGRk" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/harvestlab~website-crawl-map/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/harvestlab~website-crawl-map/runs": { "post": { "operationId": "runs-sync-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/harvestlab~website-crawl-map/run-sync": { "post": { "operationId": "run-sync-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Website URLs", "type": "array", "description": "One or more public website, page, or bare-domain URLs. Start with one site and a small page limit.", "items": { "type": "object", "properties": { "url": { "title": "URL", "type": "string", "description": "Public website or page URL to use as a crawl seed." } }, "required": [ "url" ] }, "default": [ { "url": "https://www.iana.org/help/example-domains" } ] }, "startUrl": { "title": "Start URL (CLI alias)", "type": "string", "description": "Hidden single-URL alias for startUrls." }, "url": { "title": "URL (CLI alias)", "type": "string", "description": "Hidden single-URL alias for startUrls." }, "crawlScope": { "title": "Crawl Scope", "enum": [ "same-host", "same-domain", "include-external" ], "type": "string", "description": "Same host follows only the exact hostname. Same domain includes subdomains. Include external saves outbound links but never follows them.", "default": "same-domain" }, "includeUrlPatterns": { "title": "Include URL Patterns", "type": "array", "description": "Optional shell-style patterns for saved records, for example */blog/*. Intermediate internal pages may still be traversed.", "items": { "type": "string" }, "default": [] }, "excludeUrlPatterns": { "title": "Exclude URL Patterns", "type": "array", "description": "Shell-style patterns that are neither saved nor followed, for example */logout* or *?replytocom=*.", "items": { "type": "string" }, "default": [] }, "queryParameterPolicy": { "title": "Query Parameter Policy", "enum": [ "keep-all", "drop-tracking", "strip-all" ], "type": "string", "description": "Keep all parameters, drop common analytics/tracking parameters, or strip every query parameter before deduplication.", "default": "drop-tracking" }, "assetPolicy": { "title": "Saved Link Types", "enum": [ "pages-only", "pages-and-documents", "all-links" ], "type": "string", "description": "Choose whether the inventory contains only web pages, common documents too, or every linked asset.", "default": "pages-and-documents" }, "discoverSitemaps": { "title": "Discover XML Sitemaps", "type": "boolean", "description": "Read sitemap declarations from robots.txt and conventional sitemap paths, including nested sitemap indexes.", "default": true }, "maxSitemapsPerStartUrl": { "title": "Max Sitemap Files per Website", "minimum": 1, "maximum": 500, "type": "integer", "description": "Maximum sitemap and sitemap-index documents fetched for one start URL.", "default": 50 }, "maxSitemapUrlsPerStartUrl": { "title": "Max Sitemap URLs per Website", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum in-scope URL entries merged from sitemaps for one start URL.", "default": 5000 }, "renderMode": { "title": "Page Rendering", "enum": [ "auto", "http", "browser" ], "type": "string", "description": "Auto uses fast HTTP first and renders only likely JavaScript shells. Browser renders every crawled page and costs more.", "default": "auto" }, "browserFallbackMaxPages": { "title": "Max Auto Browser Fallback Pages", "minimum": 0, "maximum": 500, "type": "integer", "description": "Maximum pages rendered after HTTP detects likely empty JavaScript shells. Ignored in browser-only mode.", "default": 10 }, "followNofollow": { "title": "Follow Nofollow Links", "type": "boolean", "description": "Record all nofollow links and also follow internal ones when enabled.", "default": true }, "maxResults": { "title": "Max URL Records", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Global maximum useful URL rows stored and charged across the run.", "default": 25 }, "maxPagesPerStartUrl": { "title": "Max Pages Crawled per Website", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Maximum unique internal pages navigated for each start URL.", "default": 8 }, "maxDepth": { "title": "Max Crawl Depth", "minimum": 0, "maximum": 30, "type": "integer", "description": "Maximum hyperlink distance from each start URL. Set 0 to fetch only the submitted page; sitemap discovery remains independent.", "default": 1 }, "maxLinksPerPage": { "title": "Max Links Processed per Page", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Maximum distinct hyperlinks parsed from one HTML page.", "default": 500 }, "maxEdges": { "title": "Max Link Graph Edges per Website", "minimum": 1, "maximum": 2000000, "type": "integer", "description": "Maximum unique source-target-anchor-rel observations retained for one website.", "default": 100000 }, "maxParentUrlsPerRecord": { "title": "Max Parent URLs Saved per Record", "minimum": 1, "maximum": 500, "type": "integer", "description": "Caps parent and anchor evidence arrays. Aggregate inbound counts remain complete up to maxEdges.", "default": 50 }, "maxConcurrency": { "title": "Max HTTP Concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Concurrent HTTP page requests. Browser concurrency is automatically capped at 3.", "default": 10 }, "requestTimeoutSecs": { "title": "Request Timeout", "minimum": 5, "maximum": 120, "type": "integer", "description": "Maximum time allowed for one HTTP or browser navigation attempt.", "default": 30 }, "maxRedirects": { "title": "Max Redirects", "minimum": 0, "maximum": 20, "type": "integer", "description": "Maximum manually validated redirect hops per HTTP request.", "default": 10 }, "maxResponseBytes": { "title": "Max Response Size", "minimum": 100000, "maximum": 20000000, "type": "integer", "description": "Maximum page or decompressed sitemap body read into memory.", "default": 5000000 }, "requestDelayMillis": { "title": "Delay Before Requests", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Polite delay before each navigation. Concurrency can still overlap requests.", "default": 1000 }, "maxRunSeconds": { "title": "Max Run Work Time", "minimum": 30, "maximum": 3600, "type": "integer", "description": "Wall-clock crawl budget. The Actor stops work early enough to preserve artifact finalization time.", "default": 600 }, "retryCount": { "title": "Network Retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Retries for timeouts, HTTP 429, and selected 5xx responses.", "default": 2 }, "saveUrlInventoryCsv": { "title": "Save URL Inventory CSV", "type": "boolean", "description": "Create URL_INVENTORY.csv in the run key-value store.", "default": true }, "saveLinkGraphCsv": { "title": "Save Link Graph CSV", "type": "boolean", "description": "Create LINK_GRAPH.csv with one row per observed hyperlink edge.", "default": true }, "saveSitemapDiffCsv": { "title": "Save Sitemap Diff CSV", "type": "boolean", "description": "Create SITEMAP_DIFF.csv with sitemap-only, crawl-only, and both classifications.", "default": true }, "generateXmlSitemap": { "title": "Generate XML Sitemap", "type": "boolean", "description": "Create GENERATED_SITEMAP.xml from internal, final-200, indexable HTML pages.", "default": false }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "Optional Apify proxy settings. Start without a proxy for public sites; enable one when a site rate-limits your run." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }