{ "openapi": "3.0.1", "info": { "title": "AI Web Scraper - Structured Data from Any Website", "description": "Scrape any website without writing selectors: name the fields you want or describe them in plain English and get clean structured rows for every page. Follow links to crawl a whole site. Export to CSV, JSON or Excel, schedule runs or use the API.", "version": "0.1", "x-build-id": "gxp1XArjgv0KIG1hA" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/automly~ai-web-scraper/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-automly-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/automly~ai-web-scraper/runs": { "post": { "operationId": "runs-sync-automly-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/automly~ai-web-scraper/run-sync": { "post": { "operationId": "run-sync-automly-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to scrape. With crawling switched on, these are also where the crawl starts.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "fields": { "title": "Fields to extract", "type": "array", "description": "Name one column per line. Add a colon and a short hint when the name alone is ambiguous, for example: price: the current price in dollars, without the currency symbol.", "items": { "type": "string" } }, "instructions": { "title": "Instructions", "type": "string", "description": "Describe in plain English what to pull out of each page. Use this on its own for free-form extraction, or alongside the field list to explain the context. Example: extract the top stories on this page, one row per story." }, "maxPages": { "title": "Maximum pages", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Hard budget for the run. Each page costs one fetch and one extraction, so this is the number that controls the bill.", "default": 10 }, "maxCrawlDepth": { "title": "Crawl depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "0 scrapes only the URLs you listed. 1 also scrapes the pages they link to, and so on. Links are followed breadth first, so the pages nearest your start URLs are scraped first.", "default": 0 }, "stayOnSite": { "title": "Stay on the same site", "type": "boolean", "description": "Follow links only within the site you started from. Turn this off with care: an open crawl can spend the whole page budget on somebody else's website.", "default": true }, "includeUrlPatterns": { "title": "Only follow URLs matching", "type": "array", "description": "Wildcard patterns, for example https://example.com/blog/*. When set, only matching links are followed.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Never follow URLs matching", "type": "array", "description": "Wildcard patterns to skip, for example */tag/* or */login*.", "items": { "type": "string" } }, "extractionMode": { "title": "How to extract", "enum": [ "selectors", "values" ], "type": "string", "description": "Reuse works out a site's structure once and applies it to the rest of that site, which is faster and much cheaper on large runs. Read every page treats each page on its own, for a run where no two pages are alike. Reuse falls back to reading a page whenever the structure it worked out does not fit.", "default": "selectors" }, "solveChallenges": { "title": "Read protected pages", "type": "boolean", "description": "On by default. Turn off to skip hard-to-read pages quickly instead of spending effort on them.", "default": true }, "maxCharactersPerPage": { "title": "Characters read per page", "minimum": 1000, "maximum": 60000, "type": "integer", "description": "How much of each page is read before extraction. Longer pages are truncated. Raise it for long articles, lower it to cut cost on pages where the answer is near the top.", "default": 12000 }, "includePageContent": { "title": "Include the page text in the output", "type": "boolean", "description": "Adds the readable text of each page to its row, next to the extracted fields. Useful for spot-checking an extraction or for feeding a RAG pipeline.", "default": false }, "effort": { "title": "Effort per page", "enum": [ "auto", "fast", "thorough" ], "type": "string", "description": "How hard to work on pages that resist. Auto keeps the run fast and steps up only for pages that need it. Fast never steps up, which is cheapest on sites that scrape easily. Thorough treats every page as difficult and is slower.", "default": "auto" }, "concurrency": { "title": "Speed", "minimum": 1, "maximum": 20, "type": "integer", "description": "Higher is faster. The default works for most sites.", "default": 5 }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Proxy settings. The default works for most runs. If pages come back empty, try a different proxy option.", "default": { "useApifyProxy": true } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }