{
  "openapi": "3.0.1",
  "info": {
    "title": "AI Training Data Scraper",
    "description": "AI Training Data Scraper converts websites into clean, semantically-chunked, vector-ready data for LLMs, RAG pipelines, and AI search. Built for documentation, tutorials, and code-heavy content, with smart chunking and rich metadata.",
    "version": "0.0",
    "x-build-id": "KalrqFPo05jqP52i5"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/blukaze~AI-Training-Data-Scraper/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-blukaze-AI-Training-Data-Scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/blukaze~AI-Training-Data-Scraper/runs": {
      "post": {
        "operationId": "runs-sync-blukaze-AI-Training-Data-Scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/blukaze~AI-Training-Data-Scraper/run-sync": {
      "post": {
        "operationId": "run-sync-blukaze-AI-Training-Data-Scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "List of URLs to begin crawling (documentation sites, forums, knowledge bases). The scraper will recursively crawl linked pages within the same domain.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "crawlerType": {
            "title": "Crawler Type",
            "enum": [
              "cheerio",
              "playwright"
            ],
            "type": "string",
            "description": "Choose based on website complexity. Use 'cheerio' for static HTML sites (10x faster), 'playwright' for JavaScript-heavy sites that require rendering.",
            "default": "cheerio"
          },
          "maxCrawlPages": {
            "title": "Max Pages to Crawl",
            "minimum": 1,
            "maximum": 10000,
            "type": "integer",
            "description": "Maximum number of pages to process. Higher values will take longer but provide more comprehensive data.",
            "default": 1
          },
          "maxCrawlDepth": {
            "title": "Max Crawl Depth",
            "minimum": 0,
            "maximum": 100,
            "type": "integer",
            "description": "How many levels deep to crawl from start URLs. Depth 0 = only start URLs, Depth 1 = start URLs + their links, etc.",
            "default": 0
          },
          "chunkingStrategy": {
            "title": "Chunking Strategy",
            "enum": [
              "fixed_token",
              "sentence_based",
              "semantic",
              "markdown_section"
            ],
            "type": "string",
            "description": "How to split content into chunks for embedding. 'semantic' is recommended for best RAG performance.",
            "default": "fixed_token"
          },
          "chunkSize": {
            "title": "Target Chunk Size",
            "minimum": 128,
            "maximum": 2048,
            "type": "integer",
            "description": "Target size in tokens. 512 is optimal for most embedding models (OpenAI, Anthropic).",
            "default": 512
          },
          "chunkOverlap": {
            "title": "Chunk Overlap",
            "minimum": 0,
            "maximum": 500,
            "type": "integer",
            "description": "Number of tokens to overlap between chunks. Prevents context loss at chunk boundaries. Recommended: 15-20% of chunk size.",
            "default": 100
          },
          "outputFormat": {
            "title": "Output Format",
            "enum": [
              "markdown",
              "plain_text",
              "json_structured",
              "vector_ready"
            ],
            "type": "string",
            "description": "Format for the extracted content. 'vector_ready' is optimized for direct ingestion into vector databases.",
            "default": "vector_ready"
          },
          "removeElements": {
            "title": "Remove Elements",
            "type": "array",
            "description": "CSS selectors for elements to remove from pages (navigation, ads, sidebars, etc.). These elements are stripped before content extraction.",
            "default": [
              "nav",
              "header",
              "footer",
              ".advertisement",
              "#cookie-banner",
              ".sidebar",
              ".nav",
              ".menu",
              ".ads",
              ".social-share",
              ".comments",
              ".related-posts"
            ],
            "items": {
              "type": "string"
            }
          },
          "includeMetadata": {
            "title": "Include Metadata",
            "type": "boolean",
            "description": "Extract and include rich metadata (author, date, keywords, language, content type). Useful for filtering during retrieval.",
            "default": false
          },
          "saveIndexPages": {
            "title": "Save Index Pages",
            "type": "boolean",
            "description": "Whether to save navigation/index pages to the dataset. These are still crawled for links even if this is false. Default is true so you see results immediately, but can be set to false for pure content-only datasets.",
            "default": false
          },
          "extractLinks": {
            "title": "Extract Links",
            "type": "boolean",
            "description": "Keep internal links within chunks for context. Can help with citation and source tracking.",
            "default": false
          },
          "saveScreenshots": {
            "title": "Save Screenshots",
            "type": "boolean",
            "description": "Save page screenshots to key-value store (useful for debugging). Only works with Playwright crawler.",
            "default": false
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Whether to respect robots.txt rules. Recommended to keep enabled for ethical scraping.",
            "default": true
          },
          "maxConcurrency": {
            "title": "Max Concurrency",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Maximum number of concurrent requests. Higher values are faster but may trigger rate limiting.",
            "default": 2
          },
          "requestTimeout": {
            "title": "Request Timeout (seconds)",
            "minimum": 10,
            "maximum": 120,
            "type": "integer",
            "description": "Maximum time to wait for a page to load before timing out.",
            "default": 30
          },
          "proxyConfiguration": {
            "title": "Proxy Configuration",
            "type": "object",
            "description": "Proxy settings for crawling. Use Apify Proxy for reliable scraping.",
            "default": {
              "useApifyProxy": true
            }
          },
          "urlPatterns": {
            "title": "URL Patterns to Include",
            "type": "array",
            "description": "Glob patterns for URLs to include. Leave empty to crawl all URLs on the domain.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "URL Patterns to Exclude",
            "type": "array",
            "description": "Glob patterns for URLs to exclude (e.g., login pages, user profiles).",
            "default": [
              "**/login**",
              "**/signup**",
              "**/register**",
              "**/cart**",
              "**/checkout**",
              "**/account**",
              "**/admin**"
            ],
            "items": {
              "type": "string"
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}