{
  "openapi": "3.0.1",
  "info": {
    "title": "Docs to Markdown + AI Embeddings → Vector DB Crawler",
    "description": "Turn any documentation site into clean Markdown, intelligently chunked content with embeddings (Azure/OpenAI), and directly upsert into MongoDB Atlas, Pinecone, Weaviate, Qdrant, or Milvus — ready for RAG, AI assistants, and semantic search in minutes.",
    "version": "0.0",
    "x-build-id": "vA5CCMjip0yhNOwMx"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/badruddeen~docs-to-markdown-ai-embeddings---vector-db-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-badruddeen-docs-to-markdown-ai-embeddings---vector-db-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/badruddeen~docs-to-markdown-ai-embeddings---vector-db-crawler/runs": {
      "post": {
        "operationId": "runs-sync-badruddeen-docs-to-markdown-ai-embeddings---vector-db-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/badruddeen~docs-to-markdown-ai-embeddings---vector-db-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-badruddeen-docs-to-markdown-ai-embeddings---vector-db-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "testMode": {
            "title": "Test Mode (internal)",
            "type": "boolean",
            "description": "Internal fast-run mode used for automated validation. When ON, limits pages, chunks, and scrolls for quick tests.",
            "default": true
          },
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "One or more full URLs to start crawling from.",
            "items": {
              "type": "string"
            },
            "default": [
              "example.com"
            ]
          },
          "linkGlobs": {
            "title": "Link Inclusion Patterns",
            "type": "array",
            "description": "Glob patterns to restrict which links are followed (e.g., /**/docs/**).",
            "items": {
              "type": "string"
            },
            "default": [
              "https://**/docs/**",
              "https://**/documentation/**",
              "https://**/guides/**",
              "https://**/guide/**",
              "https://**/manual/**",
              "https://**/reference/**",
              "https://**/api-reference/**",
              "https://**/developer/**",
              "https://**/learn/**",
              "https://**/tutorial/**",
              "https://**/how-to/**",
              "https://**/kb/**",
              "https://**/*.md"
            ]
          },
          "excludeGlobs": {
            "title": "Link Exclusion Patterns",
            "type": "array",
            "description": "Glob patterns to exclude links (takes priority over inclusion).",
            "items": {
              "type": "string"
            },
            "default": [
              "https://**/blog/**",
              "https://**/news/**",
              "https://**/changelog**",
              "https://**/releases/**",
              "https://**/v1/**",
              "https://**/v2/**",
              "https://**/archive/**"
            ]
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Obey robots.txt rules of the site.",
            "default": true
          },
          "userAgent": {
            "title": "Custom User Agent",
            "type": "string",
            "description": "User-Agent string sent during crawling.",
            "default": "Mozilla/5.0 (compatible; DocsCrawler/1.0)"
          },
          "handleScroll": {
            "title": "Handle Infinite Scroll",
            "type": "boolean",
            "description": "Scroll pages to load dynamic/lazy content.",
            "default": true
          },
          "maxScrolls": {
            "title": "Max Scrolls",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "Maximum number of scroll actions per page.",
            "default": 10
          },
          "minContentLength": {
            "title": "Minimum Content Length (characters)",
            "minimum": 200,
            "type": "integer",
            "description": "Skip pages with less readable text than this.",
            "default": 500
          },
          "chunkSize": {
            "title": "Chunk Size (characters)",
            "minimum": 400,
            "maximum": 1500,
            "type": "integer",
            "description": "Maximum characters per chunk.",
            "default": 1000
          },
          "chunkOverlap": {
            "title": "Chunk Overlap (characters)",
            "minimum": 0,
            "maximum": 1000,
            "type": "integer",
            "description": "Overlap between chunks for better context.",
            "default": 200
          },
          "maxChunksPerPage": {
            "title": "Max Chunks Per Page",
            "minimum": 1,
            "type": "integer",
            "description": "Limit chunks extracted from a single page.",
            "default": 50
          },
          "streamChunks": {
            "title": "Stream Chunks to Dataset",
            "type": "boolean",
            "description": "Push each chunk immediately (useful for huge crawls).",
            "default": false
          },
          "datasetName": {
            "title": "Dataset Name (optional)",
            "type": "string",
            "description": "Custom name for output dataset. Leave blank for default.",
            "default": ""
          },
          "generateEmbeddings": {
            "title": "Generate Embeddings",
            "type": "boolean",
            "description": "Create 1536-dim embeddings using text-embedding-3-small. ON by default: Generates embeddings for the full automated RAG/semantic pipeline - (requires Push to Vector DB also ON – otherwise actor stops with error. For normal website crawl (preview + save only to Apify Dataset): Turn this OFF (Push to Vector DB can stay OFF)",
            "default": true
          },
          "embeddingProvider": {
            "title": "Embedding Provider",
            "enum": [
              "azure",
              "openai"
            ],
            "type": "string",
            "description": "Choose Azure OpenAI or direct OpenAI API.",
            "default": "azure"
          },
          "azureOpenAiApiKey": {
            "title": "Azure OpenAI API Key",
            "type": "string",
            "description": "Your Azure OpenAI API key (will be hidden in run details for security). Required if using Azure."
          },
          "azureOpenAiEndpoint": {
            "title": "Azure OpenAI Endpoint",
            "type": "string",
            "description": "Base URL, e.g., https://your-resource.openai.azure.com"
          },
          "azureDeploymentName": {
            "title": "Azure Deployment Name",
            "type": "string",
            "description": "Name of your deployed text-embedding-3-small model."
          },
          "openAiApiKey": {
            "title": "OpenAI API Key",
            "type": "string",
            "description": "Your OpenAI API key (will be hidden in run details for security). Required if using direct OpenAI."
          },
          "embeddingBatchSize": {
            "title": "Embedding Batch Size",
            "minimum": 0,
            "maximum": 16,
            "type": "integer",
            "description": "Texts per request. 0 = auto (8 for Azure, 16 for OpenAI).",
            "default": 0
          },
          "pushToVectorDb": {
            "title": "Push to Vector DB",
            "type": "boolean",
            "description": "Upsert chunks + embeddings into a vector database. When Generate Embeddings is ON (default), this MUST also be ON for the full automated RAG pipeline. The actor will stop with an error if mismatched.",
            "default": false
          },
          "vectorDbProvider": {
            "title": "Vector DB Provider",
            "enum": [
              "pinecone",
              "weaviate",
              "qdrant",
              "milvus",
              "mongodb"
            ],
            "type": "string",
            "description": "Target database for embeddings.",
            "default": "pinecone"
          },
          "vectorDbApiKey": {
            "title": "Vector DB API Key / Token",
            "type": "string",
            "description": "Your credential for Pinecone, Weaviate, Qdrant, or Milvus/Zilliz (will be hidden in run details). Not used for MongoDB."
          },
          "vectorDbEnvironment": {
            "title": "Host / Connection String",
            "type": "string",
            "description": "Connection details:\n• Weaviate: https://your-cluster.weaviate.cloud\n• Qdrant: https://your-cluster.qdrant.io:6333\n• Milvus/Zilliz: endpoint URL\n• MongoDB Atlas: mongodb+srv://user:pass@cluster.mongodb.net \n• Pinecone: leave blank"
          },
          "vectorDbIndexName": {
            "title": "Index / Collection / Database Name",
            "type": "string",
            "description": "Storage name:\n• Pinecone: index name (auto-created)\n• Weaviate/Qdrant/Milvus: collection name (auto-created)\n• MongoDB Atlas: database name (default: 'vectors')",
            "default": "my-docs"
          },
          "vectorDbNamespace": {
            "title": "Namespace / Collection Name",
            "type": "string",
            "description": "Grouping:\n• Pinecone: namespace (optional)\n• MongoDB Atlas: collection name (default: 'chunks')\n• Leave blank for others",
            "default": ""
          },
          "pineconeBatchSize": {
            "title": "[Pinecone] Upsert Batch Size",
            "minimum": 10,
            "maximum": 500,
            "type": "integer",
            "description": "Vectors per upsert request.",
            "default": 100
          },
          "weaviateBatchSize": {
            "title": "[Weaviate] Insert Batch Size",
            "minimum": 10,
            "maximum": 200,
            "type": "integer",
            "description": "Objects per insert request.",
            "default": 50
          },
          "qdrantBatchSize": {
            "title": "[Qdrant] Upsert Batch Size",
            "minimum": 10,
            "maximum": 500,
            "type": "integer",
            "description": "Points per upsert request.",
            "default": 100
          },
          "milvusBatchSize": {
            "title": "[Milvus] Upsert Batch Size",
            "minimum": 10,
            "maximum": 500,
            "type": "integer",
            "description": "Vectors per upsert request.",
            "default": 100
          },
          "mongodbBatchSize": {
            "title": "[MongoDB] Bulk Write Batch Size",
            "minimum": 10,
            "maximum": 500,
            "type": "integer",
            "description": "Documents per bulk operation.",
            "default": 100
          },
          "logLiveSummary": {
            "title": "Log Live Summary",
            "type": "boolean",
            "description": "Show real-time progress during crawl.",
            "default": false
          },
          "visitedUrlsStoreName": {
            "title": "Visited URLs Store Name",
            "type": "string",
            "description": "Key-Value store for crawl resume support.",
            "default": "visited-urls"
          },
          "useUrlHashing": {
            "title": "Use URL Hashing for Deduplication",
            "type": "boolean",
            "description": "Prevent re-crawling same URLs across runs.",
            "default": true
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}