{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Content Crawler Pro — Markdown & RAG Chunks for AI",
    "description": "Crawl any website to clean Markdown, text or HTML for AI, LLMs and RAG. Heading-aware chunks, sitemap and llms.txt discovery, URL filters, duplicate removal, PDF link capture, rich metadata and incremental crawls. Fast HTTP or JavaScript rendering. Pay only for pages delivered.",
    "version": "1.0",
    "x-build-id": "4w6UMWnxudDsRYz2N"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/hipersoft~web-content-crawler-pro/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-hipersoft-web-content-crawler-pro",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/hipersoft~web-content-crawler-pro/runs": {
      "post": {
        "operationId": "runs-sync-hipersoft-web-content-crawler-pro",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/hipersoft~web-content-crawler-pro/run-sync": {
      "post": {
        "operationId": "run-sync-hipersoft-web-content-crawler-pro",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "urls": {
            "title": "Start URLs",
            "type": "array",
            "description": "URLs to crawl. One per line.",
            "items": {
              "type": "string"
            }
          },
          "startUrls": {
            "title": "Start URLs (request objects)",
            "type": "array",
            "description": "Advanced: start URLs as request objects.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "crawlerType": {
            "title": "Crawler type",
            "enum": [
              "http",
              "browser"
            ],
            "type": "string",
            "description": "http = fast and cheap, for sites that serve their content in the page. browser = renders JavaScript for single-page apps and client-rendered sites.",
            "default": "http"
          },
          "crawl": {
            "title": "Follow links",
            "type": "boolean",
            "description": "Crawl internal links from the start URLs (otherwise only the given URLs, plus any sitemap or llms.txt pages you enable).",
            "default": false
          },
          "maxCrawlPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 50000,
            "type": "integer",
            "description": "Maximum pages to request in total, including duplicates and pages that fail. Keeps a crawl from running away.",
            "default": 50
          },
          "maxResults": {
            "title": "Max stored pages",
            "minimum": 0,
            "type": "integer",
            "description": "Stop once this many pages are stored in the dataset (0 = no limit). Unlike Max pages, duplicates and failed pages do not count.",
            "default": 0
          },
          "maxCrawlDepth": {
            "title": "Max depth",
            "minimum": 0,
            "type": "integer",
            "description": "How many link-hops from a start URL to follow.",
            "default": 3
          },
          "sameDomainOnly": {
            "title": "Same domain only",
            "type": "boolean",
            "description": "Only follow links on the same domain as the start URL.",
            "default": true
          },
          "restrictToStartPath": {
            "title": "Stay under the start URL path",
            "type": "boolean",
            "description": "Only crawl pages under the path of a start URL: https://site.com/docs/ stays in /docs/. Off = the whole domain.",
            "default": false
          },
          "includeUrlGlobs": {
            "title": "Include URL globs",
            "type": "array",
            "description": "Only crawl URLs matching these glob patterns, e.g. https://site.com/docs/**.",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlGlobs": {
            "title": "Exclude URL globs",
            "type": "array",
            "description": "Skip URLs matching these glob patterns.",
            "items": {
              "type": "string"
            }
          },
          "useSitemaps": {
            "title": "Use sitemaps",
            "type": "boolean",
            "description": "Also discover pages from each site's sitemaps (including sitemap indexes and gzipped files declared in robots.txt). Your URL filters apply.",
            "default": false
          },
          "useLlmsTxt": {
            "title": "Use llms.txt",
            "type": "boolean",
            "description": "Look for an llms.txt file at each site's root, then crawl it and the pages it lists. Your URL filters apply.",
            "default": false
          },
          "respectRobotsTxtFile": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip pages the site's robots.txt disallows.",
            "default": false
          },
          "onlyCrawlPagesModifiedSince": {
            "title": "Only pages changed since",
            "type": "string",
            "description": "Incremental crawl: a date such as 2026-01-31 or 2026-01-31T12:00:00Z. Pages that the sitemap, the server or the page itself date earlier are not delivered and not charged. Pages with no date are delivered."
          },
          "saveMarkdown": {
            "title": "Save Markdown",
            "type": "boolean",
            "description": "Output clean Markdown (ideal for LLMs/RAG).",
            "default": true
          },
          "saveText": {
            "title": "Save plain text",
            "type": "boolean",
            "description": "Output plain text.",
            "default": true
          },
          "saveHtml": {
            "title": "Save HTML",
            "type": "boolean",
            "description": "Include the cleaned content HTML in the dataset row.",
            "default": false
          },
          "saveHtmlAsFile": {
            "title": "Save HTML as a file",
            "type": "boolean",
            "description": "Store the cleaned HTML of each page in the key-value store and add its link as htmlUrl. Keeps the dataset small.",
            "default": false
          },
          "saveLinks": {
            "title": "Save links",
            "type": "boolean",
            "description": "Include the list of links found on each page.",
            "default": false
          },
          "saveFileLinks": {
            "title": "Capture document links",
            "type": "boolean",
            "description": "List links to PDF, Word, Excel, PowerPoint, CSV, EPUB and ZIP files found on each page in fileLinks. The files themselves are not downloaded.",
            "default": true
          },
          "saveMetadata": {
            "title": "Save metadata",
            "type": "boolean",
            "description": "Add the metadata object: canonical URL, keywords, publish and update dates, image, Open Graph tags, JSON-LD and key response headers.",
            "default": true
          },
          "chunkSize": {
            "title": "RAG chunk size (characters)",
            "minimum": 0,
            "maximum": 100000,
            "type": "integer",
            "description": "Split each page into heading-aware chunks of at most this many characters, in the chunks field (about 4 characters = 1 token). 0 = off. Try 2000.",
            "default": 0
          },
          "chunkOverlap": {
            "title": "RAG chunk overlap (characters)",
            "minimum": 0,
            "maximum": 50000,
            "type": "integer",
            "description": "Characters repeated from the previous chunk, up to half the chunk size. Try 200.",
            "default": 0
          },
          "textMaxChars": {
            "title": "Max chars per page",
            "minimum": 0,
            "type": "integer",
            "description": "Truncate each page's text/markdown to this many characters (0 = no limit).",
            "default": 0
          },
          "htmlTransformer": {
            "title": "Content extraction",
            "enum": [
              "readability",
              "cleanBody",
              "none"
            ],
            "type": "string",
            "description": "readability = main article, with an automatic fallback when it would lose most of the page. cleanBody = the whole page without navigation, headers, footers and sidebars. none = the whole page body, untouched.",
            "default": "readability"
          },
          "removeBoilerplate": {
            "title": "Remove site chrome",
            "type": "boolean",
            "description": "Strip navigation, headers, footers, sidebars, breadcrumbs, share buttons and pop-ups before extraction. Not applied when Content extraction is None.",
            "default": true
          },
          "removeElementsCssSelector": {
            "title": "Remove elements (CSS)",
            "type": "string",
            "description": "CSS selector of extra elements to strip before extraction (e.g. .ads, #promo)."
          },
          "keepElementsCssSelector": {
            "title": "Keep only (CSS)",
            "type": "string",
            "description": "If set, keep only content inside the elements matching this CSS selector."
          },
          "removeCookieWarnings": {
            "title": "Remove cookie banners",
            "type": "boolean",
            "description": "Strip common cookie and consent banners.",
            "default": true
          },
          "ignoreCanonicalUrl": {
            "title": "Keep duplicate pages",
            "type": "boolean",
            "description": "By default, pages found while crawling are dropped (and not charged) when they repeat a delivered canonical URL or identical content. Start URLs are always delivered. Enable to keep every page.",
            "default": false
          },
          "dynamicContentWaitSecs": {
            "title": "Dynamic content wait (s)",
            "minimum": 0,
            "maximum": 30,
            "type": "integer",
            "description": "Browser mode: wait this many seconds for JS content to load.",
            "default": 0
          },
          "waitForSelector": {
            "title": "Wait for selector",
            "type": "string",
            "description": "Browser mode: wait until this CSS selector appears."
          },
          "maxConcurrency": {
            "title": "Concurrency",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "Max pages crawled in parallel.",
            "default": 10
          },
          "maxRequestRetries": {
            "title": "Max retries",
            "minimum": 0,
            "maximum": 10,
            "type": "integer",
            "description": "How many times to retry a page after a network or server error. Failed pages are not charged.",
            "default": 3
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (s)",
            "minimum": 5,
            "maximum": 300,
            "type": "integer",
            "description": "How long to wait for a page to load.",
            "default": 60
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Proxy configuration. Leave the default.",
            "default": {
              "useApifyProxy": true
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}