{
  "data": {
    "id": "aYG0l9s7dbB7j3gbS",
    "userId": "ZscMwFR5H7eCtWtyh",
    "name": "website-content-crawler",
    "username": "apify",
    "description": "Crawl websites and extract text content to feed AI models, LLM applications, vector databases, or RAG pipelines. The Actor supports rich formatting using Markdown, cleans the HTML, downloads files, and integrates well with 🦜🔗 LangChain, LlamaIndex, and the wider LLM ecosystem.",
    "restartOnError": true,
    "isPublic": true,
    "createdAt": "2023-03-27T15:06:04.992Z",
    "modifiedAt": "2026-10-28T01:23:34.430Z",
    "taggedBuilds": {
      "beta": {
        "buildId": "pYQWxgP95sdcV4sxU",
        "finishedAt": "2026-10-26T09:39:22.574Z",
        "buildNumberInt": 100370,
        "buildNumber": "0.1.370"
      },
      "version-0": {
        "buildId": "5i2wakgyINEu6WRZt",
        "finishedAt": "2026-10-26T11:12:33.253Z",
        "buildNumberInt": 300102,
        "buildNumber": "0.3.102"
      },
      "canary": {
        "buildId": "gAFEPYDCWQ7kIegXE",
        "finishedAt": "2026-10-27T11:14:43.272Z",
        "buildNumberInt": 200134,
        "buildNumber": "0.2.134"
      },
      "test": {
        "buildId": "gn5l5AcR7hvAl6yeE",
        "finishedAt": "2026-10-28T01:23:34.430Z",
        "buildNumberInt": 9900318,
        "buildNumber": "0.99.318"
      }
    },
    "stats": {
      "totalBuilds": 1094,
      "totalRuns": 47620392,
      "totalUsers": 172739,
      "totalUsers7Days": 5199,
      "totalUsers30Days": 13919,
      "totalUsers90Days": 25745,
      "lastRunStartedAt": "2026-10-28T08:29:23.918Z",
      "totalMetamorphs": 493,
      "publicActorRunStats30Days": {
        "ABORTED": 20559,
        "FAILED": 6958,
        "SUCCEEDED": 3453906,
        "TIMED-OUT": 96823,
        "TOTAL": 3578246
      },
      "actorReviewCount": 235,
      "actorReviewRating": 4.615066498261799,
      "bookmarkCount": 2714
    },
    "versions": [
      {
        "versionNumber": "0.0",
        "sourceType": "SOURCE_FILES",
        "buildTag": "beta"
      },
      {
        "versionNumber": "0.1",
        "sourceType": "SOURCE_FILES",
        "buildTag": "beta"
      },
      {
        "versionNumber": "0.2",
        "sourceType": "SOURCE_FILES",
        "buildTag": "canary"
      },
      {
        "versionNumber": "0.3",
        "sourceType": "SOURCE_FILES",
        "buildTag": "version-0"
      },
      {
        "versionNumber": "0.99",
        "sourceType": "GIT_REPO",
        "buildTag": "test"
      }
    ],
    "defaultRunOptions": {
      "build": "version-0",
      "timeoutSecs": 360000,
      "memoryMbytes": 8192,
      "restartOnError": true
    },
    "exampleRunInput": {
      "body": "{ \"helloWorld\": 123 }",
      "contentType": "application/json; charset=utf-8"
    },
    "categories": [
      "AI",
      "DEVELOPER_TOOLS"
    ],
    "isDeprecated": false,
    "title": "Website Content Crawler",
    "pictureUrl": "https://apify-image-uploads-prod.s3.amazonaws.com/aYG0l9s7dbB7j3gbS/PfToENkJZxahzPDu3-CleanShot_2023-03-28_at_10.40.20_2x.png",
    "seoTitle": null,
    "seoDescription": null,
    "notice": "NONE",
    "isCritical": true,
    "isGeneric": false,
    "hasNoDataset": false,
    "isSourceCodeHidden": true,
    "actorStandby": {
      "isEnabled": false,
      "disableStandbyFieldsOverride": false,
      "tenancy": "SINGLE_TENANT",
      "maxRequestsPerActorRun": 4,
      "desiredRequestsPerActorRun": 3,
      "idleTimeoutSecs": 300,
      "build": "latest",
      "memoryMbytes": 1024,
      "shouldPassActorInput": false,
      "isConsoleAuthEnabled": false,
      "isTokenlessEnabled": false
    },
    "standbyUrl": null,
    "actorPermissionLevel": "LIMITED_PERMISSIONS",
    "readmeSummary": "## Website Content Crawler\n\nA deep website crawler and content extractor for documentation, knowledge bases, help sites, blogs and other multi-page sites, optimized for downstream LLM and AI use (fine-tuning, RAG, semantic search). Built on a web-crawling engine with both headless browser and raw HTTP modes, it handles JavaScript rendering, infinite scroll, sitemaps, login-protected pages (via cookies), and file downloads (PDF, DOC/DOCX, XLS/XLSX, CSV). The Actor performs DOM cleaning to remove navigation, headers/footers, ads and cookie warnings, can expand clickable elements to reveal hidden content, and converts pages to plain text, Markdown or HTML while capturing metadata (author, language, publish date, timestamps) and screenshots. It includes anti-blocking techniques (browser fingerprinting, proxy support), supports incremental/delta crawls based on page change detection, and can optionally generate per-page AI summaries to reduce downstream token usage.\n\n## Use cases\n\n- Extract documentation, knowledge base, help-site and blog content to feed, fine-tune, or train LLMs and AI applications.  \n- Build custom customer-support chatbots that answer from a company’s website content.  \n- Generate personalized content or adapt model tone by training on a customer’s existing blogs and documentation.  \n- Enable Retrieval-Augmented Generation (RAG) pipelines that answer queries using site content as context.  \n- Large-scale summarization, translation, proofreading, or style-transfer workflows by scraping site content and processing it with LLMs.  \n- Populate vector databases / semantic search indexes and tooling (e.g., LangChain, LlamaIndex, Pinecone, Qdrant) with crawled site content for semantic retrieval.  \n- Regular incremental crawls and delta updates to detect changed pages and update downstream indexes or models.",
    "deploymentKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCe3jwR6yJ3C2l7b+V4hx4X5s+Z3XFrGmau9Yy0GC82VkTDpCKgEflGqdaLJIFmeWT1UHDwOIrCWSwk4W4DEVYQwJE3sgkoUFBGbBo0opLc/vyZFSnvjyleUwGt0vxZg2EyEgH1i9+aRirUHVNAat92Au1viClnEih1lffP0WyN4VuVgrQO0Qf+UcPfq87Adeb9UK9ZbwcWx/yhnUMwIHbWnohshyWPCWJPfiDEUFcdmFs3JfLlQZJ6y79g9zw3AFppJEIeVJhK3bHIa4mfLND049SWJxhR/prvLrCk6ZbOI2D19/CeYJNmCGxSBjPKeohIFwBvShnQZhkXIN5YkZAz \n"
  }
}