{ "openapi": "3.0.1", "info": { "title": "AI Web Scraper (Any Website)", "description": "AI Web Scraper (Any Website) SD - AI Web Scraper (Any Website) is an AI data extraction tool that extracts structured JSON from any public web page with a plain-English prompt: one row per page or per item, pagination and crawling, JS pages, Markdown and RAG chunks. Free LLM or your own key.", "version": "1.0", "x-build-id": "oDqpch1XgmFW0oOU8" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/neuro-scraper~ai-web-scraper/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-neuro-scraper-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/neuro-scraper~ai-web-scraper/runs": { "post": { "operationId": "runs-sync-neuro-scraper-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/neuro-scraper~ai-web-scraper/run-sync": { "post": { "operationId": "run-sync-neuro-scraper-ai-web-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to extract data from, e.g. https://books.toscrape.com/ or a product, article or listing page. Any public http(s) page; no login.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "prompt": { "title": "Page extraction prompt", "type": "string", "description": "What to extract, in plain English, e.g. \"Extract every product: name, price in USD, rating and product URL. Use null for fields the page does not show.\" Runs on every page. Empty prompt and no schema = the page is returned as Markdown (no AI call)." }, "instructions": { "title": "Instructions (same as the prompt)", "type": "string", "description": "Alternative name for 'Page extraction prompt' (used by GPT scraper tasks). When both are given they are joined." }, "outputMode": { "title": "Output", "enum": [ "page", "items", "answer", "markdown", "chunks" ], "type": "string", "description": "'One row per item' turns every product / post / job / row the prompt describes into its own result row. 'Free-text answer' returns the model's text in 'answer'. Markdown and chunks need no AI model.", "default": "page" }, "outputSchema": { "title": "Output schema (optional)", "type": "object", "description": "A JSON Schema the answer must follow, e.g. {\"type\": \"object\", \"properties\": {\"name\": {\"type\": \"string\"}, \"price\": {\"type\": \"number\"}}}. For 'One row per item' give the schema of ONE item. OpenAI and OpenRouter get it as structured output; other models get it in the prompt." }, "outputFields": { "title": "Output fields (optional)", "type": "array", "description": "Simpler than a schema: the keys every answer must use, e.g. title, price, availability, url.", "items": { "type": "string" } }, "schemaDescription": { "title": "Field notes (optional)", "type": "string", "description": "Extra notes about the fields, e.g. \"price without currency symbol; date as YYYY-MM-DD\"." }, "extractionMode": { "title": "Extraction mode", "enum": [ "single", "crawl", "scout", "agentic" ], "type": "string", "description": "Single: each start URL plus its 'next page' links up to 'Max pages'. Crawl: follows links of the start URL's site (link selector, globs, depth). Scout / Agentic: reads the site's sitemap (or the start page's links), lets the AI pick the URLs that match the prompt, then returns their Markdown (scout) or extracts data from them (agentic).", "default": "single" }, "maxPages": { "title": "Max pages per start URL", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Pages read per start URL: the start page and its next pages in single mode (the page's own 'next' link is followed), pages extracted in crawl / scout / agentic mode.", "default": 1 }, "maxItems": { "title": "Max results", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Stops after this many result rows in total (pages, items or chunks).", "default": 100 }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "Crawl mode: how many links away from the start URL the crawler goes (0 = start page only).", "default": 1 }, "paginationSelector": { "title": "Next-page selector (optional)", "type": "string", "description": "CSS selector of the 'next page' link, e.g. li.next a. Empty = found automatically (rel=next, or a link labelled Next / › / »)." }, "linkSelector": { "title": "Link selector (crawl)", "type": "string", "description": "Crawl mode: CSS selector of the links to follow, e.g. article h3 a. Empty = every link on the page inside the start URL's site." }, "includeUrlGlobs": { "title": "Include URLs (globs)", "type": "array", "description": "Crawl / scout / agentic: only follow URLs matching one of these globs, e.g. https://books.toscrape.com/catalogue/**. ** = anything, * = anything but /.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URLs (globs)", "type": "array", "description": "Never follow URLs matching these globs, e.g. **/login**, **/cart**.", "items": { "type": "string" } }, "skipLlmGlobs": { "title": "Pages without AI (globs)", "type": "array", "description": "Crawl mode: pages matching these globs are only used to find links - no AI call, no result (and no charge), e.g. category pages **/category/**.", "items": { "type": "string" } }, "targetSelector": { "title": "Content selector (optional)", "type": "string", "description": "CSS selector of the part of the page sent to the AI, e.g. main, #content or .product-list. Empty = the whole page minus navigation, header, footer and cookie banners." }, "removeElementsCssSelector": { "title": "Remove elements (CSS selector)", "type": "string", "description": "Elements removed before the AI reads the page, e.g. .ads, .related-posts, #comments." }, "contentScope": { "title": "Page content", "enum": [ "page", "main" ], "type": "string", "description": "'Main content only' uses an article extractor (trafilatura) and may drop product grids; keep 'Whole page' for listings.", "default": "page" }, "pageFormatInRequest": { "title": "Page format sent to the AI", "enum": [ "markdown", "html", "text" ], "type": "string", "description": "Markdown is compact; cleaned HTML keeps short class names and attributes some values live in.", "default": "markdown" }, "removeLinkUrls": { "title": "Remove link URLs", "type": "boolean", "description": "Drops link URLs from the Markdown sent to the AI (fewer tokens, but the AI can no longer return item URLs).", "default": false }, "maxContentChars": { "title": "Max page characters sent to the AI", "minimum": 1000, "maximum": 2000000, "type": "integer", "description": "Longer pages are cut at this length (about 4 characters per token). Raise it for long pages and large-context models.", "default": 40000 }, "includeMarkdown": { "title": "Add the page Markdown to each row", "type": "boolean", "description": "Also saves the page content that was sent to the AI in 'markdown' (always on in Markdown and Scout output).", "default": false }, "renderJavaScript": { "title": "JavaScript rendering", "enum": [ "auto", "always", "never" ], "type": "string", "description": "Auto loads each page over HTTP first and opens it in headless Chromium only when the HTML has (almost) no text, or when a browser-only option below is set.", "default": "auto" }, "dynamicContentWaitSecs": { "title": "Wait for dynamic content (seconds)", "minimum": 0, "maximum": 60, "type": "integer", "description": "Browser: extra seconds to wait after the page loads.", "default": 0 }, "waitForSelector": { "title": "Wait for selector (optional)", "type": "string", "description": "Browser: wait until this CSS selector appears (max 20 s), e.g. .product-card." }, "hasInfiniteScroll": { "title": "Infinite scroll", "type": "boolean", "description": "Browser: scroll to the bottom until no new content loads (up to 'Max scrolls') before reading the page.", "default": false }, "maxScrolls": { "title": "Max scrolls", "minimum": 1, "maximum": 200, "type": "integer", "description": "Browser: upper limit of scrolls per page when 'Infinite scroll' is on.", "default": 10 }, "handleCookieConsent": { "title": "Accept cookie banners", "type": "boolean", "description": "Browser: clicks a visible 'Accept' / 'Accept all' / 'I agree' button of a cookie banner. In HTTP mode cookie banners are removed from the content.", "default": true }, "deviceType": { "title": "Device", "enum": [ "desktop", "mobile" ], "type": "string", "description": "Browser: desktop or a phone (Playwright's device emulation: screen, user agent, touch). Mobile pages are often lighter.", "default": "desktop" }, "mobileDeviceModel": { "title": "Phone model (mobile)", "type": "string", "description": "Browser, mobile: a Playwright device name, e.g. iPhone 13, Pixel 7, Galaxy S9+. Empty = the newest iPhone in Playwright's list." }, "viewportWidth": { "title": "Viewport width (optional)", "minimum": 200, "maximum": 4000, "type": "integer", "description": "Browser: custom window width in pixels (default 1366 desktop / the phone's own)." }, "viewportHeight": { "title": "Viewport height (optional)", "minimum": 200, "maximum": 4000, "type": "integer", "description": "Browser: custom window height in pixels (default 900 desktop / the phone's own)." }, "saveScreenshots": { "title": "Save screenshots", "type": "boolean", "description": "Browser: saves a full-page PNG of every page in the key-value store; the link is in 'screenshotUrl'. Turns the browser on.", "default": false }, "llmProvider": { "title": "AI model provider", "enum": [ "free", "openai", "anthropic", "gemini", "openrouter", "custom" ], "type": "string", "description": "Free uses Pollinations.ai's anonymous text API: no key, a small open model (GPT-OSS 20B on 2026-10-09), about one AI call every 20-30 seconds (its anonymous limit), answers up to about 1,500 tokens (about 20-30 items per page); the page text is sent to that service. With your own key the run is much faster, handles long pages, and the AI cost is billed by that provider to you.", "default": "free" }, "llmApiKey": { "title": "AI API key", "type": "string", "description": "Your API key for the chosen provider (OpenAI sk-..., Anthropic sk-ant-..., Gemini AIza..., OpenRouter sk-or-...). Not needed for Free." }, "model": { "title": "Model (optional)", "type": "string", "description": "Model id, e.g. gpt-4.1-mini, claude-haiku-4-5, gemini-2.5-flash, openai/gpt-4o-mini. Empty = picked at run time from the provider's own model list (a fast, low-cost model)." }, "llmBaseUrl": { "title": "Base URL (OpenAI-compatible)", "type": "string", "description": "Only for 'Any OpenAI-compatible API': the base URL ending in /v1, e.g. https://api.groq.com/openai/v1, https://api.deepseek.com/v1, https://api.mistral.ai/v1." }, "temperature": { "title": "Temperature", "type": "string", "description": "0 = most deterministic (best for extraction). Models that do not take it get the request without it.", "default": "0" }, "topP": { "title": "Top P (optional)", "type": "string", "description": "Nucleus sampling, e.g. 1. Empty = the model's default." }, "frequencyPenalty": { "title": "Frequency penalty (optional)", "type": "string", "description": "OpenAI-format models only, e.g. 0. Empty = the model's default." }, "presencePenalty": { "title": "Presence penalty (optional)", "type": "string", "description": "OpenAI-format models only, e.g. 0. Empty = the model's default." }, "maxOutputTokens": { "title": "Max answer tokens (optional)", "minimum": 16, "maximum": 200000, "type": "integer", "description": "Upper limit of the answer length per page. Empty = the model's default (Anthropic: 4096)." }, "reasoningEffort": { "title": "Reasoning effort (reasoning models)", "enum": [ "", "minimal", "low", "medium", "high" ], "type": "string", "description": "OpenAI-format reasoning models (gpt-5, o-series, GPT-OSS): how long the model thinks before answering. Extraction rarely needs more than low. Models that do not take it get the request without it.", "default": "" }, "chunkSize": { "title": "Chunk size (tokens)", "minimum": 50, "maximum": 8000, "type": "integer", "description": "Chunks output: target chunk size (approx. tokens = characters / 4).", "default": 512 }, "chunkOverlap": { "title": "Chunk overlap (tokens)", "minimum": 0, "maximum": 2000, "type": "integer", "description": "Chunks output: tokens repeated from the end of one chunk at the start of the next.", "default": 64 }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skips pages the site's robots.txt disallows for all crawlers.", "default": true }, "maxConcurrency": { "title": "Start URLs in parallel", "minimum": 1, "maximum": 10, "type": "integer", "description": "How many start URLs are processed at the same time (pages of one site are read one after another, politely).", "default": 2 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Residential proxies (the default) get through to most sites; pick a country when a site shows region-specific content. The AI calls do not use the proxy.", "default": { "useApifyProxy": true, "apifyProxyGroups": [ "RESIDENTIAL" ] } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }