{
  "openapi": "3.0.1",
  "info": {
    "title": "AI Web Crawler",
    "description": "Crawl websites and extract clean, LLM-ready markdown content with stealth browser rendering, anti-bot hardening, smart content filtering, and structured metadata extraction. Built for RAG pipelines, AI agents, and data workflows.",
    "version": "0.1",
    "x-build-id": "xHhcyhbEYH9sUNl5h"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/hounderd~ai-web-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-hounderd-ai-web-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/hounderd~ai-web-crawler/runs": {
      "post": {
        "operationId": "runs-sync-hounderd-ai-web-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/hounderd~ai-web-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-hounderd-ai-web-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "List of URLs to crawl.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "crawlMode": {
            "title": "Crawl Mode",
            "enum": [
              "single",
              "bfs",
              "dfs"
            ],
            "type": "string",
            "description": "How to discover and follow links. 'single' only crawls the start URLs. 'bfs' explores level by level (breadth-first). 'dfs' goes deep before wide (depth-first).",
            "default": "single"
          },
          "maxCrawlDepth": {
            "title": "Max Crawl Depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "Maximum link-following depth from start URLs. Only used when crawl mode is BFS or DFS.",
            "default": 1
          },
          "maxCrawlPages": {
            "title": "Max Pages",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "Maximum total number of pages to crawl.",
            "default": 10
          },
          "sameDomainOnly": {
            "title": "Same Domain Only",
            "type": "boolean",
            "description": "Only follow links within the same domain as the start URL.",
            "default": true
          },
          "includeUrlPatterns": {
            "title": "Include URL Patterns",
            "type": "array",
            "description": "Only follow URLs matching these regex patterns. Leave empty to follow all URLs.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "Exclude URL Patterns",
            "type": "array",
            "description": "Skip URLs matching these regex patterns (e.g., '/login', '/signup', '\\.pdf$').",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "outputFormats": {
            "title": "Output Formats",
            "type": "array",
            "description": "Which content formats to include in the output. 'markdown' is clean markdown. 'fitMarkdown' is AI-optimized with noise removed via BM25 filtering. 'rawHtml' is the original HTML. 'cleanedHtml' has boilerplate removed. 'screenshot' is a base64 PNG.",
            "items": {
              "type": "string",
              "enum": [
                "markdown",
                "fitMarkdown",
                "rawHtml",
                "cleanedHtml",
                "screenshot"
              ],
              "enumTitles": [
                "Markdown",
                "Fit Markdown (AI-Optimized)",
                "Raw HTML",
                "Cleaned HTML",
                "Screenshot (base64 PNG)"
              ]
            },
            "default": [
              "markdown",
              "fitMarkdown"
            ]
          },
          "cssSelector": {
            "title": "Content CSS Selector",
            "type": "string",
            "description": "Restrict extraction to a specific part of the page (e.g., 'article', 'main', '#content', '.post-body'). Leave empty for full page."
          },
          "excludeSelectors": {
            "title": "Exclude CSS Selectors",
            "type": "array",
            "description": "CSS selectors for elements to remove before extraction (e.g., 'nav', 'footer', '.sidebar', '.cookie-banner').",
            "default": [
              "nav",
              "footer",
              "header",
              ".cookie-banner",
              ".consent-popup",
              "#cookie-notice"
            ],
            "items": {
              "type": "string"
            }
          },
          "waitForSelector": {
            "title": "Wait For Selector",
            "type": "string",
            "description": "Wait for this CSS selector to appear before extracting content. Useful for JS-rendered pages."
          },
          "waitForTimeout": {
            "title": "Wait Timeout (ms)",
            "minimum": 0,
            "maximum": 30000,
            "type": "integer",
            "description": "Extra time to wait after page load (milliseconds). Useful for lazy-loaded content.",
            "default": 0
          },
          "executeJavaScript": {
            "title": "Execute JavaScript",
            "type": "string",
            "description": "Custom JavaScript to execute on each page before extraction. Useful for clicking 'show more' buttons, dismissing popups, etc."
          },
          "scrollToBottom": {
            "title": "Scroll to Bottom",
            "type": "boolean",
            "description": "Scroll the full page before extraction. Triggers lazy-loaded images and infinite scroll content.",
            "default": false
          },
          "includeLinks": {
            "title": "Include Links in Markdown",
            "type": "boolean",
            "description": "Preserve hyperlinks in the markdown output. Disable for cleaner text-only output.",
            "default": true
          },
          "includeImages": {
            "title": "Include Images in Markdown",
            "type": "boolean",
            "description": "Include image references in markdown output.",
            "default": true
          },
          "includeMetadata": {
            "title": "Include Page Metadata",
            "type": "boolean",
            "description": "Extract and include page metadata (description, keywords, author, language, Open Graph data).",
            "default": true
          },
          "maxConcurrency": {
            "title": "Max Concurrency",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Maximum number of pages to crawl in parallel in standard mode. Stealth mode crawls sequentially for reliability.",
            "default": 5
          },
          "requestTimeout": {
            "title": "Page Timeout (seconds)",
            "minimum": 5,
            "maximum": 120,
            "type": "integer",
            "description": "Maximum total time to spend on one page before giving up. In stealth mode this budget includes page load, challenge waits, selector waits, and retries.",
            "default": 30
          },
          "stealthMode": {
            "title": "Stealth Mode",
            "type": "boolean",
            "description": "Enable the Camoufox-based stealth browser path to reduce browser-fingerprint detection on protected sites. Recommended for guarded targets, but some sites may still challenge or block depending on IP reputation and challenge type.",
            "default": true
          },
          "proxyConfiguration": {
            "title": "Proxy Configuration",
            "type": "object",
            "description": "Optional proxy settings. Residential proxies are recommended for sites with anti-bot protection (Cloudflare, Akamai, DataDome), but they are not required for ordinary public pages."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}