{
  "openapi": "3.0.1",
  "info": {
    "title": "Website to Knowledge Base Sync | Docs to LLM for RAG",
    "description": "Extract any public website or docs into a RAG knowledge base: the first run returns every text chunk, later runs only the chunks to upsert and IDs to delete, with stable chunk IDs. Optional vector database sync to Pinecone or Qdrant. $0.10 per 1,000 changed chunks + $0.50 per 1,000 pages checked.",
    "version": "0.1",
    "x-build-id": "pLHvx7T9G4xOX1A8e"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/aurelix~rag-site-sync/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-aurelix-rag-site-sync",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/aurelix~rag-site-sync/runs": {
      "post": {
        "operationId": "runs-sync-aurelix-rag-site-sync",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/aurelix~rag-site-sync/run-sync": {
      "post": {
        "operationId": "run-sync-aurelix-rag-site-sync",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs or sitemaps",
            "type": "array",
            "description": "Public pages to start from, e.g. ['https://docs.apify.com/academy'], or sitemap URLs ending in .xml, e.g. ['https://example.com/sitemap.xml']. Links are followed within the chosen scope. A URL without http(s):// gets https://.",
            "items": {
              "type": "string"
            }
          },
          "maxPages": {
            "title": "Max pages checked per run",
            "minimum": 1,
            "maximum": 5000,
            "type": "integer",
            "description": "Stop after this many pages are fetched in a run, e.g. 500 (pages that cannot be read count too, but only readable pages are charged). Set it above the size of the site so every known page is re-checked each run; known pages not reached keep their chunks.",
            "default": 500
          },
          "syncId": {
            "title": "Sync ID (optional)",
            "type": "string",
            "description": "Name of this sync, e.g. 'acme-docs'. Runs with the same sync ID (and chunk settings) compare against each other. Empty = derived from the start URLs, so re-running the same input just works. 1-60 letters, digits, '-' or '_'. Kept in your account's key-value store 'aurelix-rag-site-sync'."
          },
          "incremental": {
            "title": "Only changes after the first run",
            "type": "boolean",
            "description": "On (default): the first run returns every chunk, each later run only chunks to upsert and chunk IDs to delete. Off: every run returns all current chunks as upserts (plus deletes), e.g. to rebuild an index from scratch.",
            "default": true
          },
          "scope": {
            "title": "Crawl scope",
            "enum": [
              "path",
              "domain"
            ],
            "type": "string",
            "description": "'path' = only URLs under the start URL's folder (start https://example.com/docs/ syncs /docs/...), 'domain' = the whole site of each start URL.",
            "default": "path"
          },
          "includeGlobs": {
            "title": "Include URL patterns",
            "type": "array",
            "description": "Sync only URLs matching one of these patterns, e.g. ['https://docs.example.com/guides/**']. '**' matches anything, '*' anything except '/'. Empty = every in-scope URL. Pages already in the index that no longer match are deleted from it.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeGlobs": {
            "title": "Exclude URL patterns",
            "type": "array",
            "description": "Never sync URLs matching these patterns, e.g. ['**/changelog/**', '**?page=*']. Pages already in the index that now match are deleted from it.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "useSitemaps": {
            "title": "Also use the sites' sitemaps",
            "type": "boolean",
            "description": "Also seed the crawl with in-scope URLs from each start site's sitemap (robots.txt Sitemap lines or /sitemap.xml), so pages no other page links to are synced too.",
            "default": false
          },
          "chunkSize": {
            "title": "Max chunk size (characters)",
            "minimum": 300,
            "maximum": 8000,
            "type": "integer",
            "description": "Largest chunk in characters before overlap, e.g. 1500 (about 350 tokens). Chunks follow headings, then paragraph and sentence boundaries.",
            "default": 1500
          },
          "chunkOverlap": {
            "title": "Chunk overlap (characters)",
            "minimum": 0,
            "maximum": 2000,
            "type": "integer",
            "description": "Characters of the previous chunk (same section) repeated at the start of the next, e.g. 150. 0 = no overlap (most stable IDs). Changing chunk size or overlap starts a new sync with new IDs.",
            "default": 0
          },
          "vectorDatabase": {
            "title": "Push changes to a vector database",
            "enum": [
              "none",
              "pinecone",
              "qdrant"
            ],
            "type": "string",
            "description": "Optional. 'none' = results only in the dataset. 'pinecone' or 'qdrant' = also upsert and delete the changed chunks in your own index, with your own key. Qdrant needs an OpenAI key for embeddings; Pinecone works with an OpenAI key or with an index that has integrated embedding.",
            "default": "none"
          },
          "pineconeApiKey": {
            "title": "Pinecone API key",
            "type": "string",
            "description": "Your Pinecone API key. Stored encrypted by Apify, never logged or returned."
          },
          "pineconeIndexHost": {
            "title": "Pinecone index host",
            "type": "string",
            "description": "Your index's host from the Pinecone console, e.g. my-index-abc123.svc.aped-1234-a56b.pinecone.io."
          },
          "pineconeNamespace": {
            "title": "Pinecone namespace",
            "type": "string",
            "description": "Namespace to write to, e.g. 'docs'. Empty = the default namespace.",
            "default": ""
          },
          "pineconeTextField": {
            "title": "Pinecone text field (integrated embedding only)",
            "type": "string",
            "description": "Only for an index with integrated embedding and no OpenAI key: the field your index embeds, as set in its field map, e.g. 'text' or 'chunk_text'.",
            "default": "text"
          },
          "qdrantUrl": {
            "title": "Qdrant URL",
            "type": "string",
            "description": "Your Qdrant cluster URL, e.g. https://xyz-example.eu-central.aws.cloud.qdrant.io:6333."
          },
          "qdrantApiKey": {
            "title": "Qdrant API key",
            "type": "string",
            "description": "Your Qdrant API key. Stored encrypted by Apify, never logged or returned."
          },
          "qdrantCollection": {
            "title": "Qdrant collection",
            "type": "string",
            "description": "Collection name, e.g. 'docs'. Created (cosine distance) if it does not exist yet."
          },
          "openaiApiKey": {
            "title": "OpenAI API key for embeddings",
            "type": "string",
            "description": "Optional. Your OpenAI key: only new and changed chunks are embedded, billed to your OpenAI account. Without a vector database the vectors are added to each upsert record as 'embedding'. Stored encrypted by Apify, never logged or returned."
          },
          "embeddingModel": {
            "title": "Embedding model",
            "enum": [
              "text-embedding-3-small",
              "text-embedding-3-large"
            ],
            "type": "string",
            "description": "OpenAI embedding model (used only with an OpenAI key).",
            "default": "text-embedding-3-small"
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}