{ "openapi": "3.0.1", "info": { "title": "Scrape GPT - Universal AI Web Scraper Agent", "description": "AI-powered universal web scraper that works on ANY website without configuration. Extract data from e-commerce, news sites, social media, and more using intelligent LLM-based field mapping. Features JSON-first extraction, automatic pagination, anti-bot bypass, and cost-effective caching.", "version": "2.0", "x-build-id": "H0MBEwhgf1Kvur30j" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/paradox-analytics~scrape-gpt---universal-ai-web-scraper-agent/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-paradox-analytics-scrape-gpt---universal-ai-web-scraper-agent", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/paradox-analytics~scrape-gpt---universal-ai-web-scraper-agent/runs": { "post": { "operationId": "runs-sync-paradox-analytics-scrape-gpt---universal-ai-web-scraper-agent", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/paradox-analytics~scrape-gpt---universal-ai-web-scraper-agent/run-sync": { "post": { "operationId": "run-sync-paradox-analytics-scrape-gpt---universal-ai-web-scraper-agent", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls", "fields" ], "properties": { "startUrls": { "title": "URLs to Scrape", "type": "array", "description": "List of URLs to scrape. The system will automatically detect patterns and extract data.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "fields": { "title": "Fields to Extract", "type": "array", "description": "List of fields to extract (e.g., title, price, description). The LLM will generate semantic extraction patterns for these fields.", "items": { "type": "string" } }, "openaiApiKey": { "title": "🔑 OpenAI API Key", "type": "string", "description": "Your OpenAI API key (required for pattern generation). Cost: ~$0.02 per new domain, then $0.0001 for cached patterns. You can also set OPENAI_API_KEY as an environment variable." }, "useDirectLLM": { "title": "🤖 Use Direct LLM Extraction", "type": "boolean", "description": "Enable Direct LLM extraction. Extracts data directly from HTML using LLM. Recommended for maximum accuracy! Results are cached for cost efficiency.", "default": true }, "directLLMQualityMode": { "title": "Direct LLM Quality Mode", "enum": [ "conservative", "balanced", "aggressive" ], "type": "string", "description": "Quality mode for DirectLLM: 'conservative' (50% fields required, highest quality), 'balanced' (33% fields, default), 'aggressive' (10% fields, maximum items)", "default": "balanced" }, "hybridConfig": { "title": "⚙️ Hybrid System Configuration (Advanced)", "type": "object", "description": "Advanced configuration for the hybrid system", "properties": { "similarityThreshold": { "title": "Pattern Similarity Threshold", "type": "number", "description": "Threshold for pattern reuse (0.0-1.0). Higher = more strict matching. Recommended: 0.75", "default": 0.75, "minimum": 0, "maximum": 1 }, "enableLLM": { "title": "Enable LLM Pattern Generation", "type": "boolean", "description": "Use LLM to generate semantic patterns. If false, uses fallback patterns (free but less accurate).", "default": true }, "cachePatterns": { "title": "Cache Generated Patterns", "type": "boolean", "description": "Save patterns to cache for future reuse. Highly recommended for cost savings!", "default": true }, "maxContainers": { "title": "Max Containers per Page", "type": "integer", "description": "Maximum number of repeating items to extract per page", "default": 20, "minimum": 1, "maximum": 100 } } }, "proxyConfiguration": { "title": "🔗 Apify Proxy Configuration", "type": "object", "description": "Configure Apify residential proxies (recommended for production). Leave empty to use external proxy or no proxy." }, "useExternalProxy": { "title": "🌐 Use External Proxy (Bright Data, etc.)", "type": "boolean", "description": "Enable external proxy instead of Apify proxy. Requires externalProxyServer, externalProxyUsername, and externalProxyPassword.", "default": false }, "externalProxyServer": { "title": "External Proxy Server", "type": "string", "description": "External proxy server URL (e.g., http://brd.superproxy.io:33335). Required if useExternalProxy is true." }, "externalProxyUsername": { "title": "External Proxy Username", "type": "string", "description": "External proxy username. Required if useExternalProxy is true." }, "externalProxyPassword": { "title": "External Proxy Password", "type": "string", "description": "External proxy password. Required if useExternalProxy is true." }, "webUnblockerApiKey": { "title": "🌐 Web Unblocker API Key", "type": "string", "description": "Web Unblocker API key (e.g., Bright Data Web Unblocker) for bypassing Kasada/Cloudflare. This can be a Bearer token (API key) OR proxy credentials in host:port:username:password or host,port,username,password format. Used as fallback when standard proxies are blocked. Get API key from: https://brightdata.com/cp/account/api. You can also set BRIGHT_DATA_API_KEY as an environment variable." }, "webUnblockerZone": { "title": "Web Unblocker Zone", "type": "string", "description": "Bright Data Web Unblocker zone name (default: web_unlocker1)", "default": "web_unlocker1" }, "enableAutoPagination": { "title": "📄 Enable Auto-Pagination", "type": "boolean", "description": "Automatically scrape pages when pagination is detected. WARNING: Some sites have thousands of pages!", "default": false }, "maxPages": { "title": "Max Pages to Scrape", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Maximum number of pages to scrape when auto-pagination is enabled. Leave blank or set to 0 to scrape all pages. Only applies when enableAutoPagination is true.", "default": 0 }, "headless": { "title": "Headless Browser", "type": "boolean", "description": "Run browser in headless mode (recommended)", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }