{ "openapi": "3.0.1", "info": { "title": "Website Crawl Map & Sitemap Diff", "description": "Crawl websites, map every discovered URL and parent link, compare XML sitemaps, and export status, redirects, canonicals, depth, indexability, CSVs, and orphan candidates. HTTP and browser rendering. Pay per result from $0.005; diagnostics are free; x402-ready.", "version": "0.1", "x-build-id": "z6CrhvDJMl8vchXKT" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/harvestlab~website-crawl-map/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/harvestlab~website-crawl-map/runs": { "post": { "operationId": "runs-sync-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/harvestlab~website-crawl-map/run-sync": { "post": { "operationId": "run-sync-harvestlab-website-crawl-map", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Website URLs", "type": "array", "description": "One or more public website, page, or bare-domain URLs. Start with one site and a small page limit.", "items": { "type": "object", "properties": { "url": { "title": "URL", "type": "string", "description": "Public website or page URL to use as a crawl seed." } }, "required": [ "url" ] }, "default": [ { "url": "https://example.com" } ] }, "startUrl": { "title": "Start URL (CLI alias)", "type": "string", "description": "Hidden single-URL alias for startUrls." }, "url": { "title": "URL (CLI alias)", "type": "string", "description": "Hidden single-URL alias for startUrls." }, "crawlScope": { "title": "Crawl Scope", "enum": [ "same-host", "same-domain", "include-external" ], "type": "string", "description": "Same host follows only the exact hostname. Same domain includes subdomains. Include external saves outbound links but never follows them.", "default": "same-domain" }, "includeUrlPatterns": { "title": "Include URL Patterns", "type": "array", "description": "Optional shell-style patterns for saved records, for example */blog/*. Intermediate internal pages may still be traversed.", "items": { "type": "string" }, "default": [] }, "excludeUrlPatterns": { "title": "Exclude URL Patterns", "type": "array", "description": "Shell-style patterns that are neither saved nor followed, for example */logout* or *?replytocom=*.", "items": { "type": "string" }, "default": [] }, "queryParameterPolicy": { "title": "Query Parameter Policy", "enum": [ "keep-all", "drop-tracking", "strip-all" ], "type": "string", "description": "Keep all parameters, drop common analytics/tracking parameters, or strip every query parameter before deduplication.", "default": "drop-tracking" }, "assetPolicy": { "title": "Saved Link Types", "enum": [ "pages-only", "pages-and-documents", "all-links" ], "type": "string", "description": "Choose whether the inventory contains only web pages, common documents too, or every linked asset.", "default": "pages-and-documents" }, "discoverSitemaps": { "title": "Discover XML Sitemaps", "type": "boolean", "description": "Read sitemap declarations from robots.txt and conventional sitemap paths, including nested sitemap indexes.", "default": true }, "maxSitemapsPerStartUrl": { "title": "Max Sitemap Files per Website", "minimum": 1, "maximum": 500, "type": "integer", "description": "Maximum sitemap and sitemap-index documents fetched for one start URL.", "default": 50 }, "maxSitemapUrlsPerStartUrl": { "title": "Max Sitemap URLs per Website", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum in-scope URL entries merged from sitemaps for one start URL.", "default": 5000 }, "renderMode": { "title": "Page Rendering", "enum": [ "auto", "http", "browser" ], "type": "string", "description": "Auto uses fast HTTP first and renders only likely JavaScript shells. Browser renders every crawled page and costs more.", "default": "auto" }, "browserFallbackMaxPages": { "title": "Max Auto Browser Fallback Pages", "minimum": 0, "maximum": 500, "type": "integer", "description": "Maximum pages rendered after HTTP detects likely empty JavaScript shells. Ignored in browser-only mode.", "default": 10 }, "followNofollow": { "title": "Follow Nofollow Links", "type": "boolean", "description": "Record all nofollow links and also follow internal ones when enabled.", "default": true }, "maxResults": { "title": "Max URL Records", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Global maximum useful URL rows stored and charged across the run.", "default": 100 }, "maxPagesPerStartUrl": { "title": "Max Pages Crawled per Website", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Maximum unique internal pages navigated for each start URL.", "default": 50 }, "maxDepth": { "title": "Max Crawl Depth", "minimum": 0, "maximum": 30, "type": "integer", "description": "Maximum hyperlink distance from each start URL. Set 0 to fetch only the submitted page; sitemap discovery remains independent.", "default": 3 }, "maxLinksPerPage": { "title": "Max Links Processed per Page", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Maximum distinct hyperlinks parsed from one HTML page.", "default": 500 }, "maxEdges": { "title": "Max Link Graph Edges per Website", "minimum": 1, "maximum": 2000000, "type": "integer", "description": "Maximum unique source-target-anchor-rel observations retained for one website.", "default": 100000 }, "maxParentUrlsPerRecord": { "title": "Max Parent URLs Saved per Record", "minimum": 1, "maximum": 500, "type": "integer", "description": "Caps parent and anchor evidence arrays. Aggregate inbound counts remain complete up to maxEdges.", "default": 50 }, "maxConcurrency": { "title": "Max HTTP Concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Concurrent HTTP page requests. Browser concurrency is automatically capped at 3.", "default": 10 }, "requestTimeoutSecs": { "title": "Request Timeout", "minimum": 5, "maximum": 120, "type": "integer", "description": "Maximum time allowed for one HTTP or browser navigation attempt.", "default": 30 }, "maxRedirects": { "title": "Max Redirects", "minimum": 0, "maximum": 20, "type": "integer", "description": "Maximum manually validated redirect hops per HTTP request.", "default": 10 }, "maxResponseBytes": { "title": "Max Response Size", "minimum": 100000, "maximum": 20000000, "type": "integer", "description": "Maximum page or decompressed sitemap body read into memory.", "default": 5000000 }, "requestDelayMillis": { "title": "Delay Before Requests", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Polite delay before each navigation. Concurrency can still overlap requests.", "default": 1000 }, "maxRunSeconds": { "title": "Max Run Work Time", "minimum": 30, "maximum": 3600, "type": "integer", "description": "Wall-clock crawl budget. The Actor stops work early enough to preserve artifact finalization time.", "default": 600 }, "retryCount": { "title": "Network Retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Retries for timeouts, HTTP 429, and selected 5xx responses.", "default": 2 }, "saveUrlInventoryCsv": { "title": "Save URL Inventory CSV", "type": "boolean", "description": "Create URL_INVENTORY.csv in the run key-value store.", "default": true }, "saveLinkGraphCsv": { "title": "Save Link Graph CSV", "type": "boolean", "description": "Create LINK_GRAPH.csv with one row per observed hyperlink edge.", "default": true }, "saveSitemapDiffCsv": { "title": "Save Sitemap Diff CSV", "type": "boolean", "description": "Create SITEMAP_DIFF.csv with sitemap-only, crawl-only, and both classifications.", "default": true }, "generateXmlSitemap": { "title": "Generate XML Sitemap", "type": "boolean", "description": "Create GENERATED_SITEMAP.xml from internal, final-200, indexable HTML pages.", "default": false }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "Optional Apify proxy settings. Start without a proxy for public sites; enable one when a site rate-limits your run." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }