{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Embeddings for RAG - Vector DB, Pinecone, Qdrant",
    "description": "Crawl a site or docs, chunk the pages and get 1024-dim multilingual bge-m3 embeddings with stable IDs for Pinecone, Qdrant, pgvector or any vector database. No OpenAI key. From $0.20/1k chunks.",
    "version": "0.1",
    "x-build-id": "9lqbo3xR5yuVe08La"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/tidytools~website-to-embeddings/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-tidytools-website-to-embeddings",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/tidytools~website-to-embeddings/runs": {
      "post": {
        "operationId": "runs-sync-tidytools-website-to-embeddings",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/tidytools~website-to-embeddings/run-sync": {
      "post": {
        "operationId": "run-sync-tidytools-website-to-embeddings",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "urls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Main input (fill this, or `texts` / `startUrls` / `datasetId`). Website or documentation to crawl and embed, one URL per line. Bare domains (example.com) are fine; blank or invalid lines are skipped and listed in the results (not charged).",
            "items": {
              "type": "string"
            }
          },
          "texts": {
            "title": "Texts to embed (optional)",
            "type": "array",
            "description": "Embed your own texts instead of (or in addition to) crawling. Long texts are chunked too.",
            "items": {
              "type": "string"
            }
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 10000,
            "type": "integer",
            "description": "Stop crawling after this many pages.",
            "default": 50
          },
          "maxPagesPerStartUrl": {
            "title": "Max pages per start URL",
            "minimum": 0,
            "maximum": 10000,
            "type": "integer",
            "description": "With several start URLs: stop crawling one website after this many pages, so every website gets its share of Max pages. 0 = no per-website limit.",
            "default": 0
          },
          "crawlScope": {
            "title": "Crawl scope",
            "enum": [
              "path",
              "domain",
              "page"
            ],
            "type": "string",
            "description": "Which links to follow. \"Same folder\" keeps the crawl under the start URL's path; when the start page links to nothing in its own folder (docs sites that give every page its own folder, e.g. /en/getting-started/), the folder one level up is used for it, never the whole site. Values: path = Same folder as the start URL; domain = Whole website (incl. subdomains); page = Only the start URLs.",
            "default": "path"
          },
          "maxDepth": {
            "title": "Max link depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "How many clicks away from a start URL to go.",
            "default": 5
          },
          "useSitemap": {
            "title": "Use the site's sitemap",
            "enum": [
              "off",
              "seed",
              "only"
            ],
            "type": "string",
            "description": "Read sitemap.xml (found via robots.txt or common locations) and queue its in-scope pages. Finds pages that JavaScript menus hide from link discovery. Ignored when the crawl scope is \"Only the start URLs\". Values: off = Off: follow links only; seed = Seed (recommended): sitemap URLs plus links; only = Only: sitemap URLs, don't follow links.",
            "default": "seed"
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip pages the site's robots.txt disallows for crawlers. Skipped pages are not output, not charged and don't count toward Max pages. A start URL that robots.txt disallows gets one free error row (errorType robots_disallowed).",
            "default": true
          },
          "chunkSize": {
            "title": "Chunk size (characters)",
            "minimum": 200,
            "maximum": 8000,
            "type": "integer",
            "description": "Target size of each chunk; chunks are split at paragraph boundaries. 1000-2000 works well for most RAG setups.",
            "default": 1500
          },
          "chunkOverlap": {
            "title": "Chunk overlap (characters)",
            "minimum": 0,
            "maximum": 4000,
            "type": "integer",
            "description": "Characters repeated between consecutive chunks to keep context.",
            "default": 150
          },
          "includePageMarkdown": {
            "title": "Also output each page's full Markdown",
            "type": "boolean",
            "description": "Adds one extra item per page (type \"page\") with the whole Markdown and page metadata, so you don't need a separate crawl. No extra charge.",
            "default": false
          },
          "cssSelector": {
            "title": "Content selector (CSS)",
            "type": "string",
            "description": "Optional, e.g. \"main\" or \"article\", to leave menus and footers out of the embeddings."
          },
          "includeUrlPatterns": {
            "title": "Include URL patterns",
            "type": "array",
            "description": "Optional glob patterns; only matching URLs are crawled.",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "Exclude URL patterns",
            "type": "array",
            "description": "Optional glob patterns to skip.",
            "items": {
              "type": "string"
            }
          },
          "includeDocuments": {
            "title": "Embed linked documents",
            "type": "boolean",
            "description": "Also convert and embed linked PDF, Word, Excel and CSV files.",
            "default": true
          },
          "mode": {
            "title": "Page fetching (HTTP or browser)",
            "enum": [
              "auto",
              "fast",
              "browser"
            ],
            "type": "string",
            "description": "\"Auto\" uses plain HTTP and a real browser only when needed. Values: auto = Auto (recommended); fast = Fast (HTTP only); browser = Browser (JavaScript rendering).",
            "default": "auto"
          },
          "maxConcurrency": {
            "title": "Parallel pages",
            "minimum": 1,
            "maximum": 30,
            "type": "integer",
            "description": "How many pages are processed at the same time.",
            "default": 8
          },
          "httpVia": {
            "title": "Plain HTTP requests from",
            "enum": [
              "auto",
              "backend",
              "direct",
              "vm"
            ],
            "type": "string",
            "description": "Some sites block requests from data centers. Auto retries from a second network before falling back to a real browser. If both are refused, Auto also tries our second server (Oracle, different IP) before a real browser.",
            "default": "auto"
          },
          "maxConcurrentSites": {
            "title": "Parallel websites",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "With several start URLs: how many websites are crawled at the same time, so one slow website does not hold up the others. \"Parallel pages\" and Max pages are shared by all of them, and one page of Max pages is kept for every website that has not started yet, so each start page is read.",
            "default": 5
          },
          "siteTimeoutSecs": {
            "title": "Time limit per website (seconds)",
            "minimum": 0,
            "maximum": 14400,
            "type": "integer",
            "description": "Crawling one start URL (its sitemap, pages and embeddings) stops after this long. Pages embedded before the limit stay in the results (charged); pages still being read are dropped (not charged), the website gets a row with errorType \"timeout\" (not charged) and the next website starts. 0 = no limit.",
            "default": 1800
          },
          "proxyConfiguration": {
            "title": "Proxy (optional)",
            "type": "object",
            "description": "Only used for requests sent from Apify's network. Apify proxy usage is billed to your Apify account."
          },
          "removeBoilerplate": {
            "title": "Remove navigation, headers and footers",
            "type": "boolean",
            "description": "Recommended (on by default, also for API calls). Drops nav bars, site headers and footers, sidebars, breadcrumbs, cookie notices, \"Skip to main content\" links and table-of-contents widgets from the Markdown before it is chunked. Links for crawling are still taken from the whole page. Set to false to keep the full page.",
            "default": true
          },
          "removeSelectors": {
            "title": "Also remove these elements (CSS selectors)",
            "type": "string",
            "description": "Optional comma-separated CSS selectors removed from the Markdown, e.g. \".related-posts, #comments, .ad\". Works with or without the option above."
          },
          "tidyMarkdown": {
            "title": "Tidy the Markdown",
            "type": "boolean",
            "description": "Removes leftover <!doctype> lines, empty headings, 1×1 tracking-pixel images, schema.org JSON-LD code blocks and exact repeated sections before chunking (crawled pages and dataset items), so they are not embedded or charged. Set to false to embed the text unchanged.",
            "default": true
          },
          "startUrls": {
            "title": "Start URLs (request list, optional)",
            "type": "array",
            "description": "Same as the list above, in Apify's request-list format (also accepts a link to a text file of URLs). Kept for older inputs; the plain list above is recommended, especially for API calls.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "datasetId": {
            "title": "Embed another Actor's dataset (dataset ID)",
            "type": "string",
            "description": "Optional. The ID of a dataset from another run, e.g. Website Content Crawler or Website to Markdown Crawler. Every item's text is chunked and embedded: only the chunk price applies, no page fee. Can be used instead of, or together with, start URLs."
          },
          "datasetTextField": {
            "title": "Dataset text field",
            "type": "string",
            "description": "Field that holds the text to embed (dot paths allowed, e.g. \"content.body\"). Leave empty to use \"markdown\", then \"text\"."
          },
          "datasetUrlField": {
            "title": "Dataset URL field",
            "type": "string",
            "description": "Field with the page URL, used for the stable chunk IDs and the \"url\" output field.",
            "default": "url"
          },
          "datasetMaxItems": {
            "title": "Max dataset items",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "Stop after this many dataset items.",
            "default": 1000
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}