{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Intelligence Crawler",
    "description": "Website intelligence crawler: crawl a public website and get its pages as clean text and Markdown, plus chunks sized for embedding with token estimates, title, description, language and same-site links. Follows robots.txt. No model, no browser. $0.60 per 1,000 pages.",
    "version": "0.1",
    "x-build-id": "BjosbfvcFKV1uDCUF"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/dami_studio~website-intelligence-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-dami_studio-website-intelligence-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/dami_studio~website-intelligence-crawler/runs": {
      "post": {
        "operationId": "runs-sync-dami_studio-website-intelligence-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/dami_studio~website-intelligence-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-dami_studio-website-intelligence-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Public HTTP(S) URLs to crawl. Values are normalized and deduplicated.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "urls": {
            "title": "URLs",
            "type": "array",
            "description": "Additional public HTTP(S) URLs. Same-site links are followed from each seed.",
            "items": {
              "type": "string"
            }
          },
          "maxPages": {
            "title": "Maximum pages",
            "minimum": 1,
            "maximum": 1000,
            "type": "integer",
            "description": "Maximum successful or diagnostic page records per seed URL.",
            "default": 10
          },
          "maxDepth": {
            "title": "Maximum link depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "Maximum same-site link depth from each seed. Zero fetches only the seed page.",
            "default": 1
          },
          "maxConcurrency": {
            "title": "Maximum concurrency",
            "minimum": 1,
            "maximum": 10,
            "type": "integer",
            "description": "Maximum requests processed concurrently for each seed site.",
            "default": 3
          },
          "useRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Honor public robots.txt disallow and crawl-delay rules.",
            "default": true
          },
          "query": {
            "title": "Optional relevance query",
            "type": "string",
            "description": "Whitespace-separated terms used for deterministic ranking. No AI or API key is used."
          },
          "chunkSizeChars": {
            "title": "Chunk size",
            "minimum": 300,
            "maximum": 5000,
            "type": "integer",
            "description": "Target maximum characters in each deterministic content chunk.",
            "default": 1200
          },
          "chunkOverlapChars": {
            "title": "Chunk overlap",
            "minimum": 0,
            "maximum": 800,
            "type": "integer",
            "description": "Characters repeated between adjacent chunks to preserve context.",
            "default": 120
          },
          "outputFormat": {
            "title": "Output format",
            "enum": [
              "json",
              "markdown",
              "both"
            ],
            "type": "string",
            "description": "Keep clean text, Markdown, or both representations in each page record.",
            "default": "both"
          },
          "requestTimeoutSecs": {
            "title": "Request timeout",
            "minimum": 3,
            "maximum": 60,
            "type": "integer",
            "description": "Maximum time allowed for each HTTP request.",
            "default": 15
          },
          "maxResponseSizeKb": {
            "title": "Maximum response size",
            "minimum": 128,
            "maximum": 8192,
            "type": "integer",
            "description": "Maximum response body size accepted from one page in kilobytes.",
            "default": 2048
          },
          "fallbackToProxy": {
            "title": "Retry unreadable pages",
            "type": "boolean",
            "description": "Retry a page that could not be read.",
            "default": false
          },
          "proxyConfiguration": {
            "title": "Proxy configuration",
            "type": "object",
            "description": "Optional proxy configuration. Direct traffic is always tried first."
          },
          "maxPagesPerRun": {
            "title": "Maximum pages for the whole run",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "A page limit shared by all start URLs together. Leave it empty to keep only the limit per start URL."
          },
          "includeUrlGlobs": {
            "title": "Only follow links matching",
            "type": "array",
            "description": "Follow a link only when it matches one of these patterns. * matches inside one folder, ** across folders. A pattern starting with / is matched against the path and query (/docs/**), a full address against the whole URL. Start URLs are always read. Up to 20 patterns.",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlGlobs": {
            "title": "Never follow links matching",
            "type": "array",
            "description": "Never follow a link that matches one of these patterns, for example /tag/** or **?page=*. Same pattern rules as above.",
            "items": {
              "type": "string"
            }
          },
          "stayUnderStartPath": {
            "title": "Stay inside the start folder",
            "type": "boolean",
            "description": "Only follow links inside the start URL's folder: a crawl started at /docs/ stays in /docs/ and leaves /docs-old/ alone.",
            "default": false
          },
          "includeSubdomains": {
            "title": "Include subdomains",
            "type": "boolean",
            "description": "Treat subdomains of the start site as the same site, so a crawl of example.com also reads docs.example.com and help.example.com.",
            "default": false
          },
          "followNavigationLinks": {
            "title": "Follow menu, header and footer links",
            "type": "boolean",
            "description": "Also follow links in menus, headers and footers. Without it only links in the body of the page are followed, which can leave a docs site with a sidebar at one page.",
            "default": false
          },
          "skipFileLinks": {
            "title": "Skip links to files",
            "type": "boolean",
            "description": "Don't request links that end in a file type: PDFs, office files, images, media, archives, scripts, styles and feeds.",
            "default": false
          },
          "countOnlyGoodPages": {
            "title": "Only real pages use up the limit",
            "type": "boolean",
            "description": "Failed, refused and left-out pages don't count toward Maximum pages. The crawl still stops after twice that many tries.",
            "default": false
          },
          "useSitemap": {
            "title": "Start from the sitemap too",
            "type": "boolean",
            "description": "Also start from the pages in the site's sitemap (named in robots.txt, or /sitemap.xml), sitemap indexes included. A start URL ending in .xml is read as a sitemap.",
            "default": false
          },
          "useLlmsTxt": {
            "title": "Start from llms.txt too",
            "type": "boolean",
            "description": "Also start from the pages listed in the site's llms.txt. Markdown pages it links are read as Markdown.",
            "default": false
          },
          "followExternalLinks": {
            "title": "Follow links to other websites",
            "type": "boolean",
            "description": "Also read pages on other websites that the site links to. One step away only: links on those pages are not followed. Their robots.txt is always honoured, and sign-in and checkout addresses are skipped.",
            "default": false
          },
          "maxPagesPerOtherSite": {
            "title": "Pages per other website",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "How many pages to read from each other website when following links to other websites.",
            "default": 3
          },
          "requestLanguage": {
            "title": "Request language",
            "type": "string",
            "description": "The language list sent with each request, for sites that answer in the visitor's language, for example de-DE,de;q=0.9. Empty means US English."
          },
          "markdownStyle": {
            "title": "Markdown style",
            "enum": [
              "basic",
              "rich"
            ],
            "type": "string",
            "description": "Basic keeps headings, paragraphs and lists. Rich also keeps links, bold and italics, tables and code blocks. Chunks are always cut from the plain text.",
            "default": "basic"
          },
          "contentExtractor": {
            "title": "How the main text is found",
            "enum": [
              "default",
              "article",
              "wholePage"
            ],
            "type": "string",
            "description": "Main content reads the page's main or article element, or the body when there is none. Article reader picks the block with the most paragraph text. Whole page reads the body.",
            "default": "default"
          },
          "keepSelector": {
            "title": "Keep only this part (CSS selector)",
            "type": "string",
            "description": "Keep only the parts of the page that match this CSS selector, for example article .content. When nothing matches, the normal text is kept and keepSelectorMatched is false."
          },
          "removeSelector": {
            "title": "Remove these parts (CSS selector)",
            "type": "string",
            "description": "Remove the parts of the page that match this CSS selector before the text is read, for example .sidebar, .newsletter. Links are still followed from them."
          },
          "removeCookieBanners": {
            "title": "Remove cookie dialogs",
            "type": "boolean",
            "description": "Remove cookie and consent dialogs from the text, so their wording stays out of the first chunk.",
            "default": false
          },
          "dropRepeatedLines": {
            "title": "Drop repeated text blocks",
            "type": "boolean",
            "description": "Drop paragraphs and other text blocks already seen on an earlier page of the same site, such as sidebars and sign-up boxes. The text is then rebuilt block by block, and repeatedLinesDropped counts what went.",
            "default": false
          },
          "includeHtml": {
            "title": "Include cleaned HTML",
            "type": "boolean",
            "description": "Add the cleaned HTML of the content to each row as html. Rows get several times bigger.",
            "default": false
          },
          "includeContacts": {
            "title": "Include published e-mails and phone numbers",
            "type": "boolean",
            "description": "Add the e-mail addresses and tap-to-call phone numbers the site itself publishes on its pages, as emails and phones. Not read on other websites. They are personal data in many countries: keep only what you need.",
            "default": false
          },
          "saveSkippedUrls": {
            "title": "Save the links that were not followed",
            "type": "boolean",
            "description": "Save the links your rules left out, with the reason, in the key-value store record SKIPPED_URLS. Up to 10,000.",
            "default": false
          },
          "skipCanonicalDuplicates": {
            "title": "Skip copies of pages already crawled",
            "type": "boolean",
            "description": "Optional add-on. Leave out a page whose canonical address was already crawled in this run, such as copies with tracking parameters.",
            "default": false
          },
          "changedSince": {
            "title": "Only pages changed since",
            "type": "string",
            "description": "Optional add-on for repeat crawls. A date such as 2026-10-01, or a span such as 7 days. Pages the server reports as unchanged since then are left out, and sitemap entries dated earlier are not requested. Pair it with the sitemap: a page left out is not read for links when the server sends no copy."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}