{ "openapi": "3.0.1", "info": { "title": "LLM Data Pipeline Pro", "description": "Transform websites into LLM training data. Scrape, validate, deduplicate, chunk for RAG, and export to OpenAI/Anthropic/Mistral formats. Built-in PII detection and GDPR compliance. Vector DB export to Pinecone & Qdrant.", "version": "1.0", "x-build-id": "hYtHm9oXguLdoBqde" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/sanztheo~llm-data-pipeline-pro/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-sanztheo-llm-data-pipeline-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/sanztheo~llm-data-pipeline-pro/runs": { "post": { "operationId": "runs-sync-sanztheo-llm-data-pipeline-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/sanztheo~llm-data-pipeline-pro/run-sync": { "post": { "operationId": "run-sync-sanztheo-llm-data-pipeline-pro", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sourceType", "outputFormat" ], "properties": { "sourceType": { "title": "Source Type", "enum": [ "url", "dataset", "text", "pdf" ], "type": "string", "description": "Type of data source to process", "default": "url" }, "sourceUrls": { "title": "Source URLs", "type": "array", "description": "URLs to crawl and process (when sourceType is 'url')", "items": { "type": "string" } }, "sourcePdfUrls": { "title": "PDF URLs", "type": "array", "description": "URLs of PDF files to process (when sourceType is 'pdf')", "items": { "type": "string" } }, "sourceDatasetId": { "title": "Source Dataset ID", "type": "string", "description": "Apify dataset ID to process (when sourceType is 'dataset')" }, "sourceText": { "title": "Source Text", "type": "string", "description": "Raw text to process (when sourceType is 'text')" }, "outputFormat": { "title": "Output Format", "enum": [ "openai", "anthropic", "mistral", "huggingface", "raw" ], "type": "string", "description": "Target LLM format for the processed data", "default": "openai" }, "customSystemPrompt": { "title": "Custom System Prompt", "type": "string", "description": "Override the default system prompt with your own (optional). When not provided, the system prompt is automatically generated based on detected content type (technical docs, code, FAQ, tutorial, product info, or article)." }, "validationRules": { "title": "Validation Rules", "type": "object", "description": "Data quality validation options", "default": { "minContentLength": 100, "maxContentLength": 50000, "detectPII": true, "removeDuplicates": true, "semanticDedup": false, "qualityThreshold": 0.7 } }, "chunkingConfig": { "title": "Chunking Configuration", "type": "object", "description": "Text chunking settings for RAG. Strategy 'fixed' uses recursive character splitting with separators. Strategy 'semantic' respects natural text boundaries (headings, paragraphs, sentences) and keeps related content together.", "default": { "enabled": true, "strategy": "fixed", "chunkSize": 1000, "chunkOverlap": 200, "separators": [ "\n\n", "\n", ". ", " " ] } }, "vectorExport": { "title": "Vector Database Export", "type": "object", "description": "Export to vector database (optional). Set useOwnApiKey to false for managed embeddings (we handle it for you).", "default": { "enabled": false, "provider": "pinecone", "indexName": "", "namespace": "default", "useOwnApiKey": true } }, "complianceConfig": { "title": "Compliance Configuration", "type": "object", "description": "GDPR/CCPA compliance settings", "default": { "generateReport": true, "respectRobotsTxt": true, "respectAiTxt": true, "excludeSensitiveSites": true, "dataRetentionDays": 30 } }, "maxCrawlDepth": { "title": "Max Crawl Depth", "minimum": 1, "maximum": 10, "type": "integer", "description": "Maximum depth for URL crawling", "default": 2 }, "maxPages": { "title": "Max Pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Maximum number of pages to process", "default": 100 }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "Proxy settings for web crawling. Use Apify Proxy or custom proxies." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }