{ "openapi": "3.0.1", "info": { "title": "Site Crawler for Links and Tables", "description": "Crawl a website from a start address and get every page's links, download links, HTML tables, and lists as structured rows. No code, no AI, no browser, no login. Deterministic output, robots.txt obeyed by default.", "version": "0.1", "x-build-id": "pWYehcvnbBn2AtPct" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/mambalabs~site-crawler-links-tables/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-mambalabs-site-crawler-links-tables", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/mambalabs~site-crawler-links-tables/runs": { "post": { "operationId": "runs-sync-mambalabs-site-crawler-links-tables", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/mambalabs~site-crawler-links-tables/run-sync": { "post": { "operationId": "run-sync-mambalabs-site-crawler-links-tables", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "start_urls": { "title": "Start addresses", "type": "array", "description": "Web addresses to start from, up to 100. A bare domain such as example.com is read as https.", "items": { "type": "string" } }, "max_pages": { "title": "Maximum pages", "type": "string", "description": "Pages to read across the whole run, up to 10,000. Default 100.", "default": "100" }, "max_depth": { "title": "Maximum depth", "type": "string", "description": "How many links to follow from a start address. 0 reads only the start address. Default 10.", "default": "10" }, "same_domain_only": { "title": "Stay on the same site", "type": "string", "description": "yes (default): follow links only on the start address's host and its subdomains. no: also follow links to other sites, up to Maximum pages. Links to other sites are listed either way.", "default": "yes" }, "extract": { "title": "What to extract", "type": "array", "description": "Any of links, tables, lists, download_links. Default links, tables, lists. download_links adds one separate row per file link, charged as download-link-row; the download_links field on page rows is always filled at no extra charge.", "items": { "type": "string" } }, "download_extensions": { "title": "Download file extensions", "type": "array", "description": "File extensions that count as download links. Default pdf, zip, xlsx, xls, docx, doc, pptx, csv, mp3, mp4, exe, dmg, msi, pkg, gz, tar, tgz, 7z, rar, iso, apk. Files are listed, never downloaded.", "items": { "type": "string" } }, "use_sitemap": { "title": "Read the sitemap", "type": "string", "description": "yes (default): also queue the addresses in /sitemap.xml, which finds pages no link reaches. no: follow links only.", "default": "yes" }, "obey_robots": { "title": "Obey robots.txt", "type": "string", "description": "yes (default): skip pages robots.txt disallows and honor its crawl delay up to 5 seconds. no: read them, only if you have the right to.", "default": "yes" }, "max_charge_usd": { "title": "Maximum charge for this run (USD)", "type": "string", "description": "Optional. The run stops before a charge that would pass this amount. Summary rows are always returned." }, "proxy_mode": { "title": "Proxy mode", "type": "string", "description": "pool (default): the Apify proxy pool only, charged as page-row; a page that refuses it gets a free blocked row. auto: a refused page is retried on residential addresses, and a page read that way is charged as residential-page-row. residential: every page uses residential addresses and is charged as residential-page-row.", "default": "pool" }, "source_tag": { "title": "Source tag", "type": "string", "description": "Internal attribution tag. Leave empty." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }