{ "openapi": "3.0.1", "info": { "title": "AI Web Scraper - Extract Structured Data & URL to JSON", "description": "AI web scraper and AI extractor: URL to JSON. Name the fields or write a prompt, and an LLM reads each page and returns structured data. One row per list item, follows detail links, reads PDF and image documents. $0.01 per page with data. Failed pages are free.", "version": "0.1", "x-build-id": "WubhJnMaN7MQXCFPq" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/tidytools~ai-web-data-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-tidytools-ai-web-data-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/tidytools~ai-web-data-extractor/runs": { "post": { "operationId": "runs-sync-tidytools-ai-web-data-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/tidytools~ai-web-data-extractor/run-sync": { "post": { "operationId": "run-sync-tidytools-ai-web-data-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "urls": { "title": "URLs or domains", "type": "array", "description": "Paste a page URL here, then name the Fields you want or write Instructions. One URL per line; the book URL is a sample you can run now. Bare domains such as stripe.com also work. A line that is not a URL becomes a free error row; the other URLs continue. You can also use startUrls or a datasetId. Direct PDF, JPG and PNG file URLs also work.", "items": { "type": "string" } }, "startUrls": { "title": "URLs (request list, optional)", "type": "array", "description": "Same as \"URLs or domains\", in Apify's request list format (also accepts a link to a text file with URLs). Kept for existing integrations; new integrations should use \"urls\".", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "preset": { "title": "Ready-made fields", "enum": [ "property", "hotel", "menu", "recipe", "lead_directory", "product_catalog", "job_detail", "invoice", "receipt", "bank_statement" ], "type": "string", "description": "Choose a preset, then clear sample Fields. Presets supply columns and row mode; your Fields override types. Directory and catalog return one row per visible item. Catalog keeps NT$1,280, 1,280円, 12,000원 and ₩8,000 with own-product TWD, JPY or KRW evidence. A .tw, .jp or .kr domain alone does not set currency; ranges, discounts and missing prices are null. Blocked marketplaces return free rows. Use pages you own or may access." }, "fields": { "title": "Fields to extract", "type": "object", "description": "Name what to extract: {\"title\":\"string\",\"price\":\"number\",\"in_stock\":\"boolean\"}. The sample matches the book URL above; edit both for your page. Missing values return null. Clear this sample when using a preset or when you want Instructions alone to choose fields. Types: string, number, integer, boolean, array." }, "instructions": { "title": "Instructions (or just a prompt)", "type": "string", "description": "Or write one sentence, such as \"Get each product's name, price and rating\". Clear the sample Fields first if you want the AI to choose columns from this sentence. With no Fields or JSON Schema, it chooses once from the first page, reuses the columns for this run and saves them as SCHEMA. You can also use this for hints such as \"Prices in USD\"." }, "prompt": { "title": "Prompt (same as Instructions)", "type": "string", "description": "Same as \"Instructions\"; used only when Instructions is empty. Lets you paste the input JSON of Apify's AI Web Scraper (`startUrls` + `prompt`) and run it unchanged. Its crawl options (extractionMode scout/agentic, maxPagesToVisit, maxCrawlDepth) are ignored: only the start URLs are read, or turn on \"Follow links to detail pages\"." }, "outputMode": { "title": "Output", "enum": [ "page", "items" ], "type": "string", "description": "\"One row per page\": one data object per URL. \"One row per item\": for listing pages (products, jobs, search results, team members), every item on the page becomes its own row with url and itemIndex. Still charged per page. Values: page = One row per page; items = One row per item on the page (lists).", "default": "page" }, "flatOutput": { "title": "Flat columns (for Google Sheets, n8n, CSV)", "type": "boolean", "description": "Put every extracted field in its own column instead of inside \"data\". Nested objects become columns joined with _ (price.amount → price_amount), lists of values become one text cell (\"red, blue\"), lists of objects become a JSON text cell. Every row has the same columns in the same order, failed pages included (empty values, recordCount 0). recordIndex is the item's position on the page and the page title is in pageTitle. Same number of rows and same price as without it.", "default": false }, "lineItemRows": { "title": "One row per invoice or receipt line item", "type": "boolean", "description": "For invoice or receipt presets, emit one Dataset row per line item with repeated header fields. Documents without items still emit one header row. Works with Flat columns for CSV and Excel; page-extracted is charged per readable page, not per item.", "default": false }, "qualityReport": { "title": "Batch data quality report", "type": "boolean", "description": "Write DATA_QUALITY.json and DATA_QUALITY.csv with field missing counts, type error counts (a row with bad nested values, such as an invoice line item amount, counts once under its top-level field), review suggestions and completeness across saved data rows. Failed or free rows are counted separately. Quality hints are not human verification. No extra AI calls or page-extracted charges.", "default": false }, "dedupeKeyFields": { "title": "Dedupe key fields (cross-page CSV)", "maxItems": 5, "type": "array", "description": "Optional: field names that identify one entity, e.g. name and address. At the end of the run, DEDUPED_ITEMS.csv in the key-value store lists each entity once, with every source URL it was found on. Values are compared exactly after trimming, ignoring case and repeated spaces; a row missing any key field is never merged and is marked missing_key_review. The dataset keeps every row; failed pages are left out. Up to 5 fields. No extra AI calls or page-extracted charges.", "items": { "type": "string" } }, "followLinks": { "title": "Follow links to detail pages", "type": "boolean", "description": "Treat the start URLs as list pages (category, search results, index): read each one for free, then extract data from the pages it links to (e.g. every product page). The start page itself is not extracted. Each extracted page is charged as usual.", "default": false }, "followPagination": { "title": "Follow list pages automatically", "type": "boolean", "description": "Extract the starting list page, then follow visible same-domain Next or page-number links. Stops when there is no next link; it does not guess from ?page=N alone. Each successful page is one page-extracted charge, even when Output is one row per item. Default off. Use Max pages to cap charges per starting URL.", "default": false }, "maxPages": { "title": "Max pages per list", "minimum": 1, "maximum": 50, "type": "integer", "description": "Maximum pages including the starting page for each URL when Follow list pages automatically is on. Default 5; maximum 50. Each successful page is one page-extracted charge.", "default": 5 }, "linkPattern": { "title": "Link pattern", "type": "string", "description": "Which links to follow. A glob such as https://books.toscrape.com/catalogue/*/index.html (* = any text within one path segment, ** = across segments), or plain text the link must contain, such as /product/. Separate several with commas. Empty = every link to the same site (menus included, so a pattern is recommended)." }, "maxPagesPerStartUrl": { "title": "Max pages per start URL", "minimum": 1, "maximum": 1000, "type": "integer", "description": "At most this many linked pages are extracted from each start page (in page order). Caps the cost.", "default": 20 }, "jsonSchema": { "title": "JSON Schema (advanced)", "type": "object", "description": "Optional full JSON Schema (type: object) for nested output. When set, it replaces \"Fields\". In item mode it describes one item." }, "datasetId": { "title": "Dataset ID (optional)", "type": "string", "description": "Read the URLs from another Actor run's dataset, e.g. a crawler or a search results scraper. Values that are not URLs are extracted from as text." }, "datasetLabels": { "title": "Dataset labels (optional)", "minItems": 1, "maxItems": 100, "type": "array", "description": "Only extract successful rows from a classifier Dataset whose label exactly matches one of these values (case-sensitive). Requires Dataset ID; empty URLs are skipped. Omit this field to read every Dataset URL as before.", "items": { "type": "string", "minLength": 1, "pattern": "\\S" } }, "datasetField": { "title": "Dataset field", "type": "string", "description": "Field with the URLs, e.g. \"url\" or \"website\" (dot paths such as \"company.url\" work). Empty = the first URL field found." }, "mode": { "title": "Page fetching (HTTP or browser)", "enum": [ "auto", "fast", "browser" ], "type": "string", "description": "\"Auto\" reads pages over plain HTTP and switches to a real browser when a site blocks it or needs JavaScript. Values: auto = Auto (recommended); fast = Fast (HTTP only); browser = Browser (JavaScript rendering).", "default": "auto" }, "cssSelector": { "title": "Content selector (CSS)", "type": "string", "description": "Optional. Only read this part of each page, e.g. \"main\". Helps with very long pages." }, "maxConcurrency": { "title": "Parallel pages", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many pages are processed at the same time.", "default": 5 }, "itemTimeoutSecs": { "title": "Time limit per page (seconds)", "minimum": 0, "maximum": 900, "type": "integer", "description": "A page that takes longer (reading + AI) is stopped and returned as a free error row with errorType \"timeout\". In list mode, the items from the parts that finished in time are kept. 0 = no limit.", "default": 180 }, "aiModel": { "title": "AI model", "enum": [ "auto", "llama-3.3-70b", "qwen3-30b", "llama-4-scout" ], "type": "string", "description": "Same price for every model. Auto uses Llama 3.3 70B for one row per page and Qwen3 30B for list pages (one row per item), where it writes long lists about 3 times faster with the same accuracy in our tests.", "default": "auto" }, "httpVia": { "title": "Plain HTTP requests from", "enum": [ "auto", "backend", "direct", "vm" ], "type": "string", "description": "Some sites block requests from data centers. Auto retries from a second network before falling back to a real browser. If both are refused, Auto also tries our second server (Oracle, different IP) before a real browser.", "default": "auto" }, "proxyConfiguration": { "title": "Proxy (optional)", "type": "object", "description": "Only used for requests sent from Apify's network. Apify proxy usage is billed to your Apify account." }, "respectRobotsTxt": { "title": "Skip pages robots.txt disallows", "type": "boolean", "description": "On (default): pages the site's robots.txt disallows for automated readers are not read and return a free robots_disallowed row. Off: the URLs you give are read even where robots.txt disallows automated readers, as when you open them yourself. Either way we never log in and never solve CAPTCHAs; blocked pages are free.", "default": true }, "maxPdfPages": { "title": "Maximum PDF pages", "minimum": 1, "maximum": 500, "type": "integer", "description": "Read up to this many pages per PDF. Only pages with text that are sent to extraction incur one page-extracted event each.", "default": 20 }, "reviewEvidence": { "title": "Verify review ratings from page markup", "type": "boolean", "description": "For review pages with Output mode = items: read review card star markers and JSON-LD so each rating is tied to its own review (rating is null when no reviewer-specific score is found). The extra HTML read can take longer; it does not add page-extracted events.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }