{ "data": { "id": "mlbNAEkzbrejmXih5", "actId": "daEeIAgzVqlS9oXl8", "userId": "NL4ThgItAwaoBlffQ", "startedAt": "2026-10-17T06:17:42.309Z", "finishedAt": "2026-10-17T06:18:26.940Z", "status": "SUCCEEDED", "meta": { "origin": "CLI" }, "stats": { "durationMillis": 44511, "runTimeSecs": 44.511, "computeUnits": 0.04945666666666667, "imageSizeBytes": 3208543232 }, "options": { "useCache": true, "betaPackages": false, "memoryMbytes": 4096, "diskMbytes": 8192 }, "createdByOrganizationMemberUserId": "zifX0mkAFMHe8CatY", "exitCode": 0, "inputSchema": "{\n \"title\": \"Domain-Scoped Site Crawler Input\",\n \"description\": \"Enter one or more start URLs, set your crawl and content options, and run the Actor to get clean Markdown, text, or HTML per page — with metadata, links, and images. The Actor fetches each page the fastest way that works automatically (plain HTTP, browser TLS fingerprint, or a headless browser for JavaScript pages).\",\n \"type\": \"object\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"startUrls\": {\n \"title\": \"Start URLs\",\n \"type\": \"array\",\n \"description\": \"One or more URLs to start crawling from. The crawler follows links found on these pages and stays on the domain of the first start URL.\",\n \"editor\": \"requestListSources\",\n \"prefill\": [\n {\n \"url\": \"https://docs.apify.com\"\n }\n ]\n },\n \"maxCrawlDepth\": {\n \"title\": \"Max Crawl Depth\",\n \"type\": \"integer\",\n \"description\": \"How many links deep to follow from the start URLs. 0 means only the start URLs are crawled.\",\n \"default\": 10,\n \"prefill\": 1,\n \"minimum\": 0,\n \"maximum\": 100,\n \"sectionCaption\": \"Crawling\",\n \"sectionDescription\": \"Where the crawl starts and how far it goes.\"\n },\n \"maxCrawlPages\": {\n \"title\": \"Max Pages\",\n \"type\": \"integer\",\n \"description\": \"Maximum number of pages (and files) to store. The crawl stops when this limit is reached.\",\n \"default\": 20,\n \"prefill\": 3,\n \"minimum\": 1,\n \"maximum\": 100000\n },\n \"useSitemaps\": {\n \"title\": \"Use Sitemaps\",\n \"type\": \"boolean\",\n \"description\": \"Look for sitemaps (from robots.txt and /sitemap.xml) and enqueue the URLs they list. Finds pages that are not linked from the start URLs.\",\n \"default\": false,\n \"prefill\": false,\n \"sectionCaption\": \"Discovery & filters\",\n \"sectionDescription\": \"How the crawler finds pages and which URLs it keeps.\"\n },\n \"useLlmsTxt\": {\n \"title\": \"Use llms.txt\",\n \"type\": \"boolean\",\n \"description\": \"Look for /llms.txt at the start URL domains and enqueue the files listed there (common on documentation sites).\",\n \"default\": false,\n \"prefill\": false\n },\n \"respectRobotsTxtFile\": {\n \"title\": \"Respect robots.txt\",\n \"type\": \"boolean\",\n \"description\": \"Skip discovered pages that the site's robots.txt disallows. Start URLs are always crawled.\",\n \"default\": true,\n \"prefill\": true\n },\n \"crawlOnlyPagesChangedSince\": {\n \"title\": \"Crawl Only Pages Changed Since\",\n \"type\": \"string\",\n \"description\": \"Optional date (YYYY-MM-DD or full ISO date-time). Crawls only pages that may have changed since then and outputs a delta. Leave empty to crawl everything.\",\n \"default\": \"\",\n \"prefill\": \"\",\n \"editor\": \"textfield\"\n },\n \"includeUrlGlobs\": {\n \"title\": \"Include URL Patterns (globs)\",\n \"type\": \"array\",\n \"description\": \"Only crawl discovered URLs matching these glob patterns (e.g. https://example.com/docs/*). Leave empty to allow everything on the domain.\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": []\n },\n \"excludeUrlGlobs\": {\n \"title\": \"Exclude URL Patterns (globs)\",\n \"type\": \"array\",\n \"description\": \"Skip discovered URLs matching any of these glob patterns (e.g. *login*, *.pdf).\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": []\n },\n \"includeUrlRegexps\": {\n \"title\": \"Include URL Patterns (regex)\",\n \"type\": \"array\",\n \"description\": \"Only crawl discovered URLs matching at least one of these regular expressions. Combined with glob patterns (both must pass when both are set).\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"excludeUrlRegexps\": {\n \"title\": \"Exclude URL Patterns (regex)\",\n \"type\": \"array\",\n \"description\": \"Skip discovered URLs matching any of these regular expressions.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"keepUrlFragments\": {\n \"title\": \"Keep URL Fragments\",\n \"type\": \"boolean\",\n \"description\": \"Treat URLs that differ only in #fragment as different pages. Enable for single-page sites that use fragments as routes.\",\n \"default\": false,\n \"prefill\": false\n },\n \"ignoreCanonicalUrl\": {\n \"title\": \"Ignore canonical URL\",\n \"type\": \"boolean\",\n \"description\": \"When disabled, pages whose canonical URL points to a page that was already crawled are skipped as duplicates.\",\n \"default\": false,\n \"prefill\": false\n },\n \"outputFormat\": {\n \"title\": \"Output Format\",\n \"type\": \"string\",\n \"description\": \"How page content is returned. Markdown keeps headings, lists, tables, and links — best for LLMs and RAG.\",\n \"editor\": \"select\",\n \"default\": \"markdown\",\n \"prefill\": \"markdown\",\n \"enum\": [\n \"markdown\",\n \"text\",\n \"html\"\n ],\n \"enumTitles\": [\n \"Markdown\",\n \"Plain text\",\n \"Cleaned HTML\"\n ],\n \"sectionCaption\": \"Content extraction\",\n \"sectionDescription\": \"How page content is cleaned and returned.\"\n },\n \"htmlTransformer\": {\n \"title\": \"HTML Transformer\",\n \"type\": \"string\",\n \"description\": \"How the main content is isolated from navigation, footers, ads, and cookie banners.\",\n \"editor\": \"select\",\n \"default\": \"auto\",\n \"prefill\": \"auto\",\n \"enum\": [\n \"auto\",\n \"trafilatura\",\n \"readability\",\n \"basic\",\n \"none\"\n ],\n \"enumTitles\": [\n \"Auto (recommended) — best content, boilerplate removed\",\n \"Trafilatura (article/text extraction)\",\n \"Readability (Mozilla article extractor)\",\n \"Basic cleanup (remove nav/header/footer only)\",\n \"No cleaning (keep the full page)\"\n ]\n },\n \"includeImages\": {\n \"title\": \"Include Images in Markdown\",\n \"type\": \"boolean\",\n \"description\": \"Add image references to the markdown field. Image URLs are always collected in the imageUrls field.\",\n \"default\": false,\n \"prefill\": false\n },\n \"chunkMarkdown\": {\n \"title\": \"Split Content into Chunks\",\n \"type\": \"boolean\",\n \"description\": \"Emit a chunks array per page — heading-aware Markdown pieces ready for embeddings. Code blocks and tables are never split, each piece carries its heading path and character count, and consecutive pieces overlap so context is kept at the boundaries.\",\n \"default\": false,\n \"prefill\": false\n },\n \"chunkMaxChars\": {\n \"title\": \"Chunk Size (characters)\",\n \"type\": \"integer\",\n \"description\": \"Target size of each chunk when chunkMarkdown is on. Code blocks and tables stay whole even when larger than this.\",\n \"minimum\": 200,\n \"maximum\": 10000,\n \"default\": 2000,\n \"prefill\": 2000\n },\n \"chunkOverlapChars\": {\n \"title\": \"Chunk Overlap (characters)\",\n \"type\": \"integer\",\n \"description\": \"Approximate overlap between consecutive chunks when chunkMarkdown is on, so context is not lost at chunk boundaries.\",\n \"minimum\": 0,\n \"maximum\": 1000,\n \"default\": 200,\n \"prefill\": 200\n },\n \"keepElementsCssSelector\": {\n \"title\": \"Keep HTML Elements (CSS selector)\",\n \"type\": \"string\",\n \"description\": \"Keep only the elements matching this CSS selector (e.g. article, main .content) before extraction. Leave empty to use the whole page.\",\n \"default\": \"\",\n \"prefill\": \"\",\n \"editor\": \"textfield\"\n },\n \"removeElementsCssSelector\": {\n \"title\": \"Remove HTML Elements (CSS selector)\",\n \"type\": \"string\",\n \"description\": \"Remove elements matching this CSS selector (e.g. .sidebar, .ad, nav) before extraction.\",\n \"default\": \"\",\n \"prefill\": \"\",\n \"editor\": \"textfield\"\n },\n \"parsePdfText\": {\n \"title\": \"Extract PDF Text\",\n \"type\": \"boolean\",\n \"description\": \"When a crawled URL serves a PDF, extract its text into the text field as a file record instead of skipping it.\",\n \"default\": true,\n \"prefill\": true\n },\n \"clickElementsCssSelector\": {\n \"title\": \"Click Elements (CSS selector)\",\n \"type\": \"string\",\n \"description\": \"Click elements matching this CSS selector (e.g. [aria-expanded=\\\"false\\\"]) before extraction to expand collapsed content.\",\n \"default\": \"\",\n \"prefill\": \"\",\n \"editor\": \"textfield\",\n \"sectionCaption\": \"Rendered pages\",\n \"sectionDescription\": \"Options for content that only appears after JavaScript runs.\"\n },\n \"takeScreenshots\": {\n \"title\": \"Take Page Screenshots\",\n \"type\": \"boolean\",\n \"description\": \"Render every stored page in the browser and capture a full-page JPEG (quality 80) of it, putting its public URL in the screenshot field. Screenshots are held in memory until pushed and add roughly 0.3-1 MB and a fraction of a second per page; screenshot capture always runs after the page is rendered.\",\n \"default\": false,\n \"prefill\": false\n },\n \"customHttpHeaders\": {\n \"title\": \"Custom HTTP Headers\",\n \"type\": \"array\",\n \"description\": \"Extra request headers for all engines and discovery requests, as a list of {\\\"key\\\", \\\"value\\\"} pairs (e.g. an Authorization token or a specific User-Agent for sites that ask for one). Transport/framing and cookie headers (Host, Cookie, Content-Length, Connection, ...) are rejected so the actor stays cookieless.\",\n \"editor\": \"json\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"key\": {\n \"type\": \"string\",\n \"title\": \"Header name\",\n \"description\": \"HTTP header name, e.g. Authorization or Accept-Language.\"\n },\n \"value\": {\n \"type\": \"string\",\n \"title\": \"Header value\",\n \"description\": \"Header value sent with every request.\"\n }\n },\n \"required\": [\n \"key\",\n \"value\"\n ]\n },\n \"default\": [],\n \"prefill\": [],\n \"sectionCaption\": \"Advanced\",\n \"sectionDescription\": \"Request headers and run diagnostics.\"\n },\n \"storeSkippedUrls\": {\n \"title\": \"Store Skipped URLs\",\n \"type\": \"boolean\",\n \"description\": \"Save a SKIPPED_URLS record to the key-value store listing skipped URLs grouped by reason (robots, blocked, non-HTML, errors, duplicates).\",\n \"default\": false,\n \"prefill\": false\n }\n },\n \"required\": [\n \"startUrls\"\n ]\n}", "readme": "# Domain-Scoped Site Crawler for Markdown Datasets\n\nGive it one or more start URLs and it crawls the site the way a documentation index should be crawled: it follows same-domain links, isolates the main content of every page, and returns clean **Markdown**, **text**, or **HTML** alongside per-page metadata — title, description, author, publish dates, canonical URL, links, and image URLs. Built for LLM training sets, RAG pipelines, knowledge bases, and content migrations.\n\n## What this actor does\n\n- **Fetches pages with the right engine automatically.** Every page starts with fast raw HTTP; hosts that reject plain clients get a browser TLS fingerprint (`curl_cffi`), and pages that render only with JavaScript are loaded in a headless browser — no engine choice needed.\n- **Finds more pages.** Follows links up to your depth limit, and can also enqueue URLs from `sitemap.xml` / robots.txt sitemaps and from `/llms.txt`.\n- **Respects robots.txt** by default: discovered URLs that the site disallows are skipped (start URLs are always crawled).\n- **Extracts the meaningful content.** An extraction cascade (Trafilatura → Readability → basic cleanup) removes navigation, footers, sidebars, ads, and cookie dialogs, while preserving headings, tables, lists, code blocks, and links.\n- **Prepares content for RAG.** Every record carries an approximate token count, and you can switch on heading-aware `chunks` (code blocks and tables never split, context-preserving overlap) that drop straight into an embedding pipeline.\n- **Returns rich metadata per page**: canonical URL, author, published/modified dates, keywords, OpenGraph, JSON-LD, image URLs, page links, word count, and a content hash.\n- **Handles PDFs**: when a crawled URL serves a PDF, its text is extracted into the record as a file row — never raw binary.\n- **Stores Markdown and plain-text files**: pages served as `text/markdown` or plain text (for example the `.md`/`.txt` files listed in an `llms.txt`) are stored with their content in `text` and `markdown`, and their links are followed when depth allows.\n- **Skips non-pages cleanly**: RSS/XML feeds, JSON, images, archives and other binary responses are skipped with a reason (see `storeSkippedUrls`).\n- **Supports incremental re-crawls**: `crawlOnlyPagesChangedSince` produces a delta of changed pages using HTTP validators, sitemap `lastmod`, and content hashes.\n- **Cuts crawl waste**: canonical duplicates are skipped, pages are pushed as they finish, retries and waits are bounded per URL, and an adaptive per-domain pacer slows down (bounded) only when a host actually throttles.\n- **Handles late and lazy content**: a bounded wait after load lets client-rendered pages finish, and pages that declare lazy images or infinite scroll are scrolled automatically (bounded) so that content is captured.\n- **Optional screenshots**: every stored page can be rendered in the browser and captured as a full-page JPEG with a public URL.\n- **Optional request headers**: send extra headers (e.g. a token for a public API or a specific User-Agent) on every request; cookie and transport headers are refused, so the actor stays cookieless.\n- **Works cookieless** for any public website — no logins, no API keys.\n\n## Output per page\n\nEvery record is pushed with empty values omitted (no `null`, no empty strings). A page record contains:\n\n| Field | Description |\n|---|---|\n| `url` | URL that was requested |\n| `loadedUrl` | Final URL after redirects |\n| `title` | OpenGraph title, JSON-LD headline, or `` |\n| `description` | Meta description / OpenGraph description |\n| `languageCode` | Language from `lang`, OpenGraph locale, or hreflang |\n| `author` | JSON-LD author, meta author, citation author, or byline |\n| `publishedDate` | Normalized ISO date: JSON-LD `datePublished`, `article:published_time`, `citation_date`, or `<time datetime>` |\n| `modifiedDate` | Normalized ISO date: JSON-LD `dateModified`, `article:modified_time`, or `last-modified` |\n| `canonicalUrl` | `<link rel=\"canonical\">`, `og:url`, or JSON-LD URL |\n| `keywords` | Meta keywords or JSON-LD keywords, as an array |\n| `imageUrls` | Every image URL found (OpenGraph, Twitter, `<img>`), absolute and unique |\n| `links` | Every link found on the page, absolute and unique |\n| `openGraph` | Raw `og:*` and `twitter:*` meta tags as an object |\n| `jsonLd` | Parsed `application/ld+json` blocks (including `@graph`) |\n| `text` | Clean plain text of the page content |\n| `markdown` | Content as Markdown (only when `outputFormat` is `markdown`) |\n| `html` | Cleaned content HTML (only when `outputFormat` is `html`) |\n| `wordCount` | Word count of `text` |\n| `tokenCount` | Approximate LLM tokens in the returned content (characters ÷ 4) |\n| `chunks` | Heading-aware Markdown pieces for embeddings (only with `chunkMarkdown`); each has `index`, `text`, `charCount`, and `headingPath` when the piece sits under headings |\n| `screenshot` | Public URL of the page's full-page JPEG (only with `takeScreenshots`) |\n| `contentHash` | SHA-256 of the normalized text — use it to detect unchanged pages |\n| `depth` | Crawl depth (`0` = start URL) |\n| `httpStatusCode` | HTTP status of the response |\n| `loadedTime` | ISO 8601 UTC timestamp when the page was fetched |\n| `referrerUrl` | URL of the page that linked here (start URLs have none) |\n| `recordType` | `page` for HTML pages, `file` for extracted documents |\n| `fileType` | `pdf` on file records |\n| `pageCount` | Number of pages, on PDF records |\n\n### Example output row (truncated)\n\n```json\n{\n \"url\": \"https://example.com/blog/release-notes\",\n \"loadedUrl\": \"https://example.com/blog/release-notes\",\n \"title\": \"Release notes — Example\",\n \"description\": \"What changed in the latest release.\",\n \"author\": \"Jane Developer\",\n \"publishedDate\": \"2026-09-12T00:00:00Z\",\n \"canonicalUrl\": \"https://example.com/blog/release-notes\",\n \"languageCode\": \"en\",\n \"imageUrls\": [\"https://example.com/media/hero.png\"],\n \"links\": [\"https://example.com/blog\", \"https://example.com/download\"],\n \"text\": \"What changed in the latest release...\",\n \"markdown\": \"## What changed\\n\\n- Faster startup\\n\",\n \"depth\": 1,\n \"httpStatusCode\": 200,\n \"loadedTime\": \"2026-10-16T12:00:05.123Z\",\n \"referrerUrl\": \"https://example.com/blog\",\n \"wordCount\": 412,\n \"contentHash\": \"8f2c1d…\",\n \"recordType\": \"page\"\n}\n```\n\nPDF record example:\n\n```json\n{\n \"url\": \"https://example.com/whitepaper.pdf\",\n \"loadededUrl\": \"https://example.com/whitepaper.pdf\",\n \"title\": \"Whitepaper 2026\",\n \"text\": \"Executive summary ...\",\n \"fileType\": \"pdf\",\n \"recordType\": \"file\",\n \"wordCount\": 5321\n}\n```\n\n## Input\n\nPicking a fetch engine, parallelism, timeouts, retries, rate limits, and proxies are auto-decided per page and per domain — no tuning needed.\n\n| Field | Type | Default | Description |\n|---|---|---|---|\n| `startUrls` | array | – | Pages where the crawl begins. Required. Accepts URL objects or plain URL strings. |\n| `maxCrawlDepth` | integer | `10` | How many links deep to follow (`0` = start URLs only). |\n| `maxCrawlPages` | integer | `20` | Total pages and files to store; the crawl stops at this limit. |\n| `useSitemaps` | boolean | `false` | Enqueue URLs from robots.txt sitemaps and `/sitemap.xml` (sitemap indexes are followed). |\n| `useLlmsTxt` | boolean | `false` | Enqueue the files linked from `/llms.txt`. |\n| `respectRobotsTxtFile` | boolean | `true` | Skip discovered URLs disallowed by robots.txt. Start URLs are always crawled. |\n| `crawlOnlyPagesChangedSince` | string | – | Date (`YYYY-MM-DD` or full ISO) for incremental crawls; pages unchanged since then are not stored and the output is a delta. |\n| `includeUrlGlobs` | array | `[]` | Only crawl discovered URLs matching these glob patterns. |\n| `excludeUrlGlobs` | array | `[]` | Skip discovered URLs matching these glob patterns. |\n| `includeUrlRegexps` | array | `[]` | Only crawl discovered URLs matching at least one regex. |\n| `excludeUrlRegexps` | array | `[]` | Skip discovered URLs matching any regex. |\n| `keepUrlFragments` | boolean | `false` | Treat `#fragment` variants as separate pages (useful for fragment-routed SPAs). |\n| `ignoreCanonicalUrl` | boolean | `false` | When off, a page whose canonical URL was already crawled is skipped as a duplicate. |\n| `outputFormat` | string | `markdown` | `markdown`, `text`, or `html`. |\n| `htmlTransformer` | string | `auto` | `auto` picks Trafilatura or Readability per page; `trafilatura`, `readability`, `basic`, and `none` force a strategy. |\n| `includeImages` | boolean | `false` | Add image references to Markdown (`imageUrls` is filled regardless). |\n| `keepElementsCssSelector` | string | – | Keep only matching elements before extraction (e.g. `article, main`). |\n| `removeElementsCssSelector` | string | – | Remove matching elements before extraction (e.g. `.sidebar, .ad`). |\n| `parsePdfText` | boolean | `true` | Extract text from linked PDFs into file records instead of skipping them. |\n| `chunkMarkdown` | boolean | `false` | Emit a `chunks` array per page for RAG/embeddings. |\n| `chunkMaxChars` | integer | `2000` | Target chunk size in characters (200–10000); code blocks and tables stay whole. |\n| `chunkOverlapChars` | integer | `200` | Overlap between consecutive chunks (0–1000). |\n| `takeScreenshots` | boolean | `false` | Render every stored page in the browser, store a full-page JPEG, and return its URL in `screenshot`. |\n| `customHttpHeaders` | array | `[]` | Extra request headers as `{\"key\", \"value\"}` pairs, applied to every engine. Cookie/transport headers are refused. |\n| `clickElementsCssSelector` | string | – | Click these elements first (e.g. `[aria-expanded=\"false\"]`) to expand collapsed content. |\n| `storeSkippedUrls` | boolean | `false` | Save a `SKIPPED_URLS` key-value record with skipped URLs grouped by reason. |\n\n### Examples\n\nCrawl a documentation site to Markdown with metadata:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://docs.example.com\" }],\n \"maxCrawlPages\": 500,\n \"maxCrawlDepth\": 5,\n \"outputFormat\": \"markdown\"\n}\n```\n\nOnly one section of a site, using glob and regex filters:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com\" }],\n \"includeUrlGlobs\": [\"https://example.com/docs/*\"],\n \"excludeUrlRegexps\": [\"/v[0-9]+/\", \"/changelog/\"]\n}\n```\n\nWeekly incremental re-crawl (delta of changed pages):\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com/blog\" }],\n \"useSitemaps\": true,\n \"crawlOnlyPagesChangedSince\": \"2026-10-01\"\n}\n```\n\nFind pages through the sitemap only, without following links:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com\" }],\n \"useSitemaps\": true,\n \"maxCrawlDepth\": 0,\n \"maxCrawlPages\": 2000\n}\n```\n\nCrawl a documentation site straight into embedding-ready chunks:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://docs.example.com\" }],\n \"maxCrawlPages\": 500,\n \"chunkMarkdown\": true,\n \"chunkMaxChars\": 1500,\n \"chunkOverlapChars\": 200\n}\n```\n\n## Use cases\n\n- **LLM training and fine-tuning data** — turn documentation, blogs, and help centers into structured Markdown with source metadata\n- **RAG pipelines and semantic search** — per-page content with optional heading-aware `chunks` and `tokenCount`, plus canonical URLs, dates, and hashes for clean indexing and re-indexing\n- **Knowledge-base and chatbot grounding** — keep a site's content current with scheduled incremental crawls\n- **Content migration and archival** — export a whole site to Markdown while keeping authorship and publication dates\n- **Competitive and market research** — compare content across sites with consistent fields per page\n\n## FAQ\n\n**Do I need to choose a crawler engine?**\nNo. The actor starts every page with fast raw HTTP, retries blocked hosts with a browser TLS fingerprint (`curl_cffi`), and loads pages that render only with JavaScript in a headless browser. Parallelism, timeouts, retries, and pacing are also chosen automatically per page and per domain, so the input surface only contains options that change which data you get.\n\n**How does the extraction cascade pick content?**\nWith `htmlTransformer` set to `auto`, the actor runs Trafilatura and Readability on each page, keeps the candidate that preserves the most structure (headings, tables, lists, images), and falls back to full basic-cleaned content when both would drop most of the page. Use `none` to keep the page almost untouched.\n\n**Does it respect robots.txt?**\nYes, `respectRobotsTxtFile` is on by default: discovered URLs that robots.txt disallows are skipped and listed under the `robots` skip reason. Start URLs are always crawled. Crawl-delay directives are not applied; the adaptive per-domain pacer keeps request rates polite automatically.\n\n**How do sitemaps work?**\nWith `useSitemaps` enabled, the actor reads the `Sitemap:` entries from robots.txt and `/sitemap.xml`, follows sitemap indexes, and enqueues matching same-domain URLs at depth 1. Combined with `maxCrawlDepth: 0` this crawls exactly the sitemap's URL set.\n\n**What happens with JavaScript-heavy sites?**\nPages whose raw HTML is an empty SPA shell are automatically rendered in a headless browser. After load the actor waits a bounded couple of seconds for late content, waits again briefly if the DOM still looks empty, and scrolls (bounded) only when the page declares lazy images or infinite scroll, so client-rendered and lazy content is captured. `clickElementsCssSelector` expands collapsed sections when you need it.\n\n**What happens with PDFs, feeds, and other files?**\nPDFs are converted to text and stored as `recordType: \"file\"` rows with `fileType: \"pdf\"`. Markdown and plain-text files (`text/markdown`, `text/plain`) are stored as pages with their raw content in `text`/`markdown` (JSON/XML payloads served as text are still skipped). RSS/XML feeds, JSON, and binary responses (images, archives) are not stored as pages; each skip is counted and visible in the run log, and in the `SKIPPED_URLS` record when `storeSkippedUrls` is enabled.\n\n**How do incremental crawls work?**\nSet `crawlOnlyPagesChangedSince` to a date. The actor keeps ETag/Last-Modified validators and content hashes in a persistent key-value store, filters sitemap entries by `lastmod`, and stores only pages that changed — the output is a delta, not a full snapshot. Keep previous results if you need the complete site.\n\n**How does chunking work?**\nWith `chunkMarkdown` on, each page also gets a `chunks` array: heading-aware Markdown pieces up to `chunkMaxChars` characters (200–10000) with `chunkOverlapChars` (0–1000) of overlap so context survives the boundaries. Code fences and tables are never split, every piece knows its `headingPath` (`Guide > Install > Linux`) and `charCount`, and the pieces are in reading order. `tokenCount` on the page is an approximation (characters ÷ 4) — good for budgeting, not a tokenizer.\n\n**Can I take screenshots?**\nYes, with `takeScreenshots`. Every stored page is rendered in the headless browser and gets a full-page JPEG (quality 80) in the run's key-value store; the record's `screenshot` field is its public URL. Expect roughly 0.3–1 MB and a fraction of a second per page; the bytes are held only until the page is pushed.\n\n**Can I send custom headers?**\n`customHttpHeaders` accepts a list of `{\"key\", \"value\"}` pairs and applies them to page requests and to robots.txt/sitemap/llms.txt lookups on every engine — useful for public APIs that want a token or sites that ask for a contact User-Agent. Cookie and transport/framing headers (`Cookie`, `Host`, `Content-Length`, `Connection`, ...) are refused on purpose: the actor never carries logins.\n\n**What happens when a site throttles the crawl?**\nThe actor watches each domain: after connection timeouts or 429/503 responses it halves the parallel requests for that domain (down to 1) and adds a small delay (up to 2 seconds), then heals as pages succeed again. Throttled pages are requeued a bounded number of times, so a hostile host can't stall the run.\n\n**Which fields are filled for every page?**\nContent-derived fields (`url`, `loadedUrl`, `depth`, `httpStatusCode`, `loadedTime`, `recordType`, and one of `text`/`markdown`/`html`) are always present. Metadata fields (`author`, `publishedDate`, `keywords`, and so on) appear only when the page declares them, and empty values are never pushed.\n\n**When do results appear in the dataset?**\nPages are pushed as soon as they finish, so you can watch results grow while the crawl is running. If a run is interrupted, everything already stored is kept.\n\n**Are image and link URLs usable directly?**\n`imageUrls` and `links` contain absolute `http(s)` URLs exactly as the page declares them. No rehosting is done.\n\n**Can it crawl pages behind a login?**\nNo. The actor only works with publicly accessible pages and never requires credentials.\n\n**What is out of scope?**\nLogin-protected content and AI summaries are intentionally not implemented — crawling behind a login needs credentials this actor refuses to require, and summaries need an external LLM key. Chunking and token counting are built in (`chunkMarkdown`, `tokenCount`); for anything more, the dataset stays atomic per page with a `contentHash` you can diff.\n\n**Are the links in Markdown absolute?**\nYes. Relative links and images are resolved against the page URL before converting to Markdown.\n\n**Is this affiliated with the websites it crawls?**\nNo. This is an independent, third-party actor that crawls publicly accessible pages; it isn't operated by or affiliated with any site it's pointed at. Check the target site's terms before scraping.\n", "actorDefinition": { "actorSpecification": 1, "name": "domain-scoped-site-crawler-for-markdown-datasets", "title": "Domain-Scoped Site Crawler for Markdown Datasets", "description": "Crawl a website from start URLs, staying on the same domain, and get clean Markdown, text, or HTML per page with title, author, dates, links, and images. Adaptive engine, sitemaps, robots.txt, PDFs, incremental crawls.", "version": "1.1", "buildTag": "latest", "readme": "# Domain-Scoped Site Crawler for Markdown Datasets\n\nGive it one or more start URLs and it crawls the site the way a documentation index should be crawled: it follows same-domain links, isolates the main content of every page, and returns clean **Markdown**, **text**, or **HTML** alongside per-page metadata — title, description, author, publish dates, canonical URL, links, and image URLs. Built for LLM training sets, RAG pipelines, knowledge bases, and content migrations.\n\n## What this actor does\n\n- **Fetches pages with the right engine automatically.** Every page starts with fast raw HTTP; hosts that reject plain clients get a browser TLS fingerprint (`curl_cffi`), and pages that render only with JavaScript are loaded in a headless browser — no engine choice needed.\n- **Finds more pages.** Follows links up to your depth limit, and can also enqueue URLs from `sitemap.xml` / robots.txt sitemaps and from `/llms.txt`.\n- **Respects robots.txt** by default: discovered URLs that the site disallows are skipped (start URLs are always crawled).\n- **Extracts the meaningful content.** An extraction cascade (Trafilatura → Readability → basic cleanup) removes navigation, footers, sidebars, ads, and cookie dialogs, while preserving headings, tables, lists, code blocks, and links.\n- **Prepares content for RAG.** Every record carries an approximate token count, and you can switch on heading-aware `chunks` (code blocks and tables never split, context-preserving overlap) that drop straight into an embedding pipeline.\n- **Returns rich metadata per page**: canonical URL, author, published/modified dates, keywords, OpenGraph, JSON-LD, image URLs, page links, word count, and a content hash.\n- **Handles PDFs**: when a crawled URL serves a PDF, its text is extracted into the record as a file row — never raw binary.\n- **Stores Markdown and plain-text files**: pages served as `text/markdown` or plain text (for example the `.md`/`.txt` files listed in an `llms.txt`) are stored with their content in `text` and `markdown`, and their links are followed when depth allows.\n- **Skips non-pages cleanly**: RSS/XML feeds, JSON, images, archives and other binary responses are skipped with a reason (see `storeSkippedUrls`).\n- **Supports incremental re-crawls**: `crawlOnlyPagesChangedSince` produces a delta of changed pages using HTTP validators, sitemap `lastmod`, and content hashes.\n- **Cuts crawl waste**: canonical duplicates are skipped, pages are pushed as they finish, retries and waits are bounded per URL, and an adaptive per-domain pacer slows down (bounded) only when a host actually throttles.\n- **Handles late and lazy content**: a bounded wait after load lets client-rendered pages finish, and pages that declare lazy images or infinite scroll are scrolled automatically (bounded) so that content is captured.\n- **Optional screenshots**: every stored page can be rendered in the browser and captured as a full-page JPEG with a public URL.\n- **Optional request headers**: send extra headers (e.g. a token for a public API or a specific User-Agent) on every request; cookie and transport headers are refused, so the actor stays cookieless.\n- **Works cookieless** for any public website — no logins, no API keys.\n\n## Output per page\n\nEvery record is pushed with empty values omitted (no `null`, no empty strings). A page record contains:\n\n| Field | Description |\n|---|---|\n| `url` | URL that was requested |\n| `loadedUrl` | Final URL after redirects |\n| `title` | OpenGraph title, JSON-LD headline, or `<title>` |\n| `description` | Meta description / OpenGraph description |\n| `languageCode` | Language from `lang`, OpenGraph locale, or hreflang |\n| `author` | JSON-LD author, meta author, citation author, or byline |\n| `publishedDate` | Normalized ISO date: JSON-LD `datePublished`, `article:published_time`, `citation_date`, or `<time datetime>` |\n| `modifiedDate` | Normalized ISO date: JSON-LD `dateModified`, `article:modified_time`, or `last-modified` |\n| `canonicalUrl` | `<link rel=\"canonical\">`, `og:url`, or JSON-LD URL |\n| `keywords` | Meta keywords or JSON-LD keywords, as an array |\n| `imageUrls` | Every image URL found (OpenGraph, Twitter, `<img>`), absolute and unique |\n| `links` | Every link found on the page, absolute and unique |\n| `openGraph` | Raw `og:*` and `twitter:*` meta tags as an object |\n| `jsonLd` | Parsed `application/ld+json` blocks (including `@graph`) |\n| `text` | Clean plain text of the page content |\n| `markdown` | Content as Markdown (only when `outputFormat` is `markdown`) |\n| `html` | Cleaned content HTML (only when `outputFormat` is `html`) |\n| `wordCount` | Word count of `text` |\n| `tokenCount` | Approximate LLM tokens in the returned content (characters ÷ 4) |\n| `chunks` | Heading-aware Markdown pieces for embeddings (only with `chunkMarkdown`); each has `index`, `text`, `charCount`, and `headingPath` when the piece sits under headings |\n| `screenshot` | Public URL of the page's full-page JPEG (only with `takeScreenshots`) |\n| `contentHash` | SHA-256 of the normalized text — use it to detect unchanged pages |\n| `depth` | Crawl depth (`0` = start URL) |\n| `httpStatusCode` | HTTP status of the response |\n| `loadedTime` | ISO 8601 UTC timestamp when the page was fetched |\n| `referrerUrl` | URL of the page that linked here (start URLs have none) |\n| `recordType` | `page` for HTML pages, `file` for extracted documents |\n| `fileType` | `pdf` on file records |\n| `pageCount` | Number of pages, on PDF records |\n\n### Example output row (truncated)\n\n```json\n{\n \"url\": \"https://example.com/blog/release-notes\",\n \"loadedUrl\": \"https://example.com/blog/release-notes\",\n \"title\": \"Release notes — Example\",\n \"description\": \"What changed in the latest release.\",\n \"author\": \"Jane Developer\",\n \"publishedDate\": \"2026-09-12T00:00:00Z\",\n \"canonicalUrl\": \"https://example.com/blog/release-notes\",\n \"languageCode\": \"en\",\n \"imageUrls\": [\"https://example.com/media/hero.png\"],\n \"links\": [\"https://example.com/blog\", \"https://example.com/download\"],\n \"text\": \"What changed in the latest release...\",\n \"markdown\": \"## What changed\\n\\n- Faster startup\\n\",\n \"depth\": 1,\n \"httpStatusCode\": 200,\n \"loadedTime\": \"2026-10-16T12:00:05.123Z\",\n \"referrerUrl\": \"https://example.com/blog\",\n \"wordCount\": 412,\n \"contentHash\": \"8f2c1d…\",\n \"recordType\": \"page\"\n}\n```\n\nPDF record example:\n\n```json\n{\n \"url\": \"https://example.com/whitepaper.pdf\",\n \"loadededUrl\": \"https://example.com/whitepaper.pdf\",\n \"title\": \"Whitepaper 2026\",\n \"text\": \"Executive summary ...\",\n \"fileType\": \"pdf\",\n \"recordType\": \"file\",\n \"wordCount\": 5321\n}\n```\n\n## Input\n\nPicking a fetch engine, parallelism, timeouts, retries, rate limits, and proxies are auto-decided per page and per domain — no tuning needed.\n\n| Field | Type | Default | Description |\n|---|---|---|---|\n| `startUrls` | array | – | Pages where the crawl begins. Required. Accepts URL objects or plain URL strings. |\n| `maxCrawlDepth` | integer | `10` | How many links deep to follow (`0` = start URLs only). |\n| `maxCrawlPages` | integer | `20` | Total pages and files to store; the crawl stops at this limit. |\n| `useSitemaps` | boolean | `false` | Enqueue URLs from robots.txt sitemaps and `/sitemap.xml` (sitemap indexes are followed). |\n| `useLlmsTxt` | boolean | `false` | Enqueue the files linked from `/llms.txt`. |\n| `respectRobotsTxtFile` | boolean | `true` | Skip discovered URLs disallowed by robots.txt. Start URLs are always crawled. |\n| `crawlOnlyPagesChangedSince` | string | – | Date (`YYYY-MM-DD` or full ISO) for incremental crawls; pages unchanged since then are not stored and the output is a delta. |\n| `includeUrlGlobs` | array | `[]` | Only crawl discovered URLs matching these glob patterns. |\n| `excludeUrlGlobs` | array | `[]` | Skip discovered URLs matching these glob patterns. |\n| `includeUrlRegexps` | array | `[]` | Only crawl discovered URLs matching at least one regex. |\n| `excludeUrlRegexps` | array | `[]` | Skip discovered URLs matching any regex. |\n| `keepUrlFragments` | boolean | `false` | Treat `#fragment` variants as separate pages (useful for fragment-routed SPAs). |\n| `ignoreCanonicalUrl` | boolean | `false` | When off, a page whose canonical URL was already crawled is skipped as a duplicate. |\n| `outputFormat` | string | `markdown` | `markdown`, `text`, or `html`. |\n| `htmlTransformer` | string | `auto` | `auto` picks Trafilatura or Readability per page; `trafilatura`, `readability`, `basic`, and `none` force a strategy. |\n| `includeImages` | boolean | `false` | Add image references to Markdown (`imageUrls` is filled regardless). |\n| `keepElementsCssSelector` | string | – | Keep only matching elements before extraction (e.g. `article, main`). |\n| `removeElementsCssSelector` | string | – | Remove matching elements before extraction (e.g. `.sidebar, .ad`). |\n| `parsePdfText` | boolean | `true` | Extract text from linked PDFs into file records instead of skipping them. |\n| `chunkMarkdown` | boolean | `false` | Emit a `chunks` array per page for RAG/embeddings. |\n| `chunkMaxChars` | integer | `2000` | Target chunk size in characters (200–10000); code blocks and tables stay whole. |\n| `chunkOverlapChars` | integer | `200` | Overlap between consecutive chunks (0–1000). |\n| `takeScreenshots` | boolean | `false` | Render every stored page in the browser, store a full-page JPEG, and return its URL in `screenshot`. |\n| `customHttpHeaders` | array | `[]` | Extra request headers as `{\"key\", \"value\"}` pairs, applied to every engine. Cookie/transport headers are refused. |\n| `clickElementsCssSelector` | string | – | Click these elements first (e.g. `[aria-expanded=\"false\"]`) to expand collapsed content. |\n| `storeSkippedUrls` | boolean | `false` | Save a `SKIPPED_URLS` key-value record with skipped URLs grouped by reason. |\n\n### Examples\n\nCrawl a documentation site to Markdown with metadata:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://docs.example.com\" }],\n \"maxCrawlPages\": 500,\n \"maxCrawlDepth\": 5,\n \"outputFormat\": \"markdown\"\n}\n```\n\nOnly one section of a site, using glob and regex filters:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com\" }],\n \"includeUrlGlobs\": [\"https://example.com/docs/*\"],\n \"excludeUrlRegexps\": [\"/v[0-9]+/\", \"/changelog/\"]\n}\n```\n\nWeekly incremental re-crawl (delta of changed pages):\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com/blog\" }],\n \"useSitemaps\": true,\n \"crawlOnlyPagesChangedSince\": \"2026-10-01\"\n}\n```\n\nFind pages through the sitemap only, without following links:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://example.com\" }],\n \"useSitemaps\": true,\n \"maxCrawlDepth\": 0,\n \"maxCrawlPages\": 2000\n}\n```\n\nCrawl a documentation site straight into embedding-ready chunks:\n\n```json\n{\n \"startUrls\": [{ \"url\": \"https://docs.example.com\" }],\n \"maxCrawlPages\": 500,\n \"chunkMarkdown\": true,\n \"chunkMaxChars\": 1500,\n \"chunkOverlapChars\": 200\n}\n```\n\n## Use cases\n\n- **LLM training and fine-tuning data** — turn documentation, blogs, and help centers into structured Markdown with source metadata\n- **RAG pipelines and semantic search** — per-page content with optional heading-aware `chunks` and `tokenCount`, plus canonical URLs, dates, and hashes for clean indexing and re-indexing\n- **Knowledge-base and chatbot grounding** — keep a site's content current with scheduled incremental crawls\n- **Content migration and archival** — export a whole site to Markdown while keeping authorship and publication dates\n- **Competitive and market research** — compare content across sites with consistent fields per page\n\n## FAQ\n\n**Do I need to choose a crawler engine?**\nNo. The actor starts every page with fast raw HTTP, retries blocked hosts with a browser TLS fingerprint (`curl_cffi`), and loads pages that render only with JavaScript in a headless browser. Parallelism, timeouts, retries, and pacing are also chosen automatically per page and per domain, so the input surface only contains options that change which data you get.\n\n**How does the extraction cascade pick content?**\nWith `htmlTransformer` set to `auto`, the actor runs Trafilatura and Readability on each page, keeps the candidate that preserves the most structure (headings, tables, lists, images), and falls back to full basic-cleaned content when both would drop most of the page. Use `none` to keep the page almost untouched.\n\n**Does it respect robots.txt?**\nYes, `respectRobotsTxtFile` is on by default: discovered URLs that robots.txt disallows are skipped and listed under the `robots` skip reason. Start URLs are always crawled. Crawl-delay directives are not applied; the adaptive per-domain pacer keeps request rates polite automatically.\n\n**How do sitemaps work?**\nWith `useSitemaps` enabled, the actor reads the `Sitemap:` entries from robots.txt and `/sitemap.xml`, follows sitemap indexes, and enqueues matching same-domain URLs at depth 1. Combined with `maxCrawlDepth: 0` this crawls exactly the sitemap's URL set.\n\n**What happens with JavaScript-heavy sites?**\nPages whose raw HTML is an empty SPA shell are automatically rendered in a headless browser. After load the actor waits a bounded couple of seconds for late content, waits again briefly if the DOM still looks empty, and scrolls (bounded) only when the page declares lazy images or infinite scroll, so client-rendered and lazy content is captured. `clickElementsCssSelector` expands collapsed sections when you need it.\n\n**What happens with PDFs, feeds, and other files?**\nPDFs are converted to text and stored as `recordType: \"file\"` rows with `fileType: \"pdf\"`. Markdown and plain-text files (`text/markdown`, `text/plain`) are stored as pages with their raw content in `text`/`markdown` (JSON/XML payloads served as text are still skipped). RSS/XML feeds, JSON, and binary responses (images, archives) are not stored as pages; each skip is counted and visible in the run log, and in the `SKIPPED_URLS` record when `storeSkippedUrls` is enabled.\n\n**How do incremental crawls work?**\nSet `crawlOnlyPagesChangedSince` to a date. The actor keeps ETag/Last-Modified validators and content hashes in a persistent key-value store, filters sitemap entries by `lastmod`, and stores only pages that changed — the output is a delta, not a full snapshot. Keep previous results if you need the complete site.\n\n**How does chunking work?**\nWith `chunkMarkdown` on, each page also gets a `chunks` array: heading-aware Markdown pieces up to `chunkMaxChars` characters (200–10000) with `chunkOverlapChars` (0–1000) of overlap so context survives the boundaries. Code fences and tables are never split, every piece knows its `headingPath` (`Guide > Install > Linux`) and `charCount`, and the pieces are in reading order. `tokenCount` on the page is an approximation (characters ÷ 4) — good for budgeting, not a tokenizer.\n\n**Can I take screenshots?**\nYes, with `takeScreenshots`. Every stored page is rendered in the headless browser and gets a full-page JPEG (quality 80) in the run's key-value store; the record's `screenshot` field is its public URL. Expect roughly 0.3–1 MB and a fraction of a second per page; the bytes are held only until the page is pushed.\n\n**Can I send custom headers?**\n`customHttpHeaders` accepts a list of `{\"key\", \"value\"}` pairs and applies them to page requests and to robots.txt/sitemap/llms.txt lookups on every engine — useful for public APIs that want a token or sites that ask for a contact User-Agent. Cookie and transport/framing headers (`Cookie`, `Host`, `Content-Length`, `Connection`, ...) are refused on purpose: the actor never carries logins.\n\n**What happens when a site throttles the crawl?**\nThe actor watches each domain: after connection timeouts or 429/503 responses it halves the parallel requests for that domain (down to 1) and adds a small delay (up to 2 seconds), then heals as pages succeed again. Throttled pages are requeued a bounded number of times, so a hostile host can't stall the run.\n\n**Which fields are filled for every page?**\nContent-derived fields (`url`, `loadedUrl`, `depth`, `httpStatusCode`, `loadedTime`, `recordType`, and one of `text`/`markdown`/`html`) are always present. Metadata fields (`author`, `publishedDate`, `keywords`, and so on) appear only when the page declares them, and empty values are never pushed.\n\n**When do results appear in the dataset?**\nPages are pushed as soon as they finish, so you can watch results grow while the crawl is running. If a run is interrupted, everything already stored is kept.\n\n**Are image and link URLs usable directly?**\n`imageUrls` and `links` contain absolute `http(s)` URLs exactly as the page declares them. No rehosting is done.\n\n**Can it crawl pages behind a login?**\nNo. The actor only works with publicly accessible pages and never requires credentials.\n\n**What is out of scope?**\nLogin-protected content and AI summaries are intentionally not implemented — crawling behind a login needs credentials this actor refuses to require, and summaries need an external LLM key. Chunking and token counting are built in (`chunkMarkdown`, `tokenCount`); for anything more, the dataset stays atomic per page with a `contentHash` you can diff.\n\n**Are the links in Markdown absolute?**\nYes. Relative links and images are resolved against the page URL before converting to Markdown.\n\n**Is this affiliated with the websites it crawls?**\nNo. This is an independent, third-party actor that crawls publicly accessible pages; it isn't operated by or affiliated with any site it's pointed at. Check the target site's terms before scraping.\n", "input": { "title": "Domain-Scoped Site Crawler Input", "description": "Enter one or more start URLs, set your crawl and content options, and run the Actor to get clean Markdown, text, or HTML per page — with metadata, links, and images. The Actor fetches each page the fastest way that works automatically (plain HTTP, browser TLS fingerprint, or a headless browser for JavaScript pages).", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "One or more URLs to start crawling from. The crawler follows links found on these pages and stays on the domain of the first start URL.", "editor": "requestListSources", "prefill": [ { "url": "https://docs.apify.com" } ] }, "maxCrawlDepth": { "title": "Max Crawl Depth", "type": "integer", "description": "How many links deep to follow from the start URLs. 0 means only the start URLs are crawled.", "default": 10, "prefill": 1, "minimum": 0, "maximum": 100, "sectionCaption": "Crawling", "sectionDescription": "Where the crawl starts and how far it goes." }, "maxCrawlPages": { "title": "Max Pages", "type": "integer", "description": "Maximum number of pages (and files) to store. The crawl stops when this limit is reached.", "default": 20, "prefill": 3, "minimum": 1, "maximum": 100000 }, "useSitemaps": { "title": "Use Sitemaps", "type": "boolean", "description": "Look for sitemaps (from robots.txt and /sitemap.xml) and enqueue the URLs they list. Finds pages that are not linked from the start URLs.", "default": false, "prefill": false, "sectionCaption": "Discovery & filters", "sectionDescription": "How the crawler finds pages and which URLs it keeps." }, "useLlmsTxt": { "title": "Use llms.txt", "type": "boolean", "description": "Look for /llms.txt at the start URL domains and enqueue the files listed there (common on documentation sites).", "default": false, "prefill": false }, "respectRobotsTxtFile": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip discovered pages that the site's robots.txt disallows. Start URLs are always crawled.", "default": true, "prefill": true }, "crawlOnlyPagesChangedSince": { "title": "Crawl Only Pages Changed Since", "type": "string", "description": "Optional date (YYYY-MM-DD or full ISO date-time). Crawls only pages that may have changed since then and outputs a delta. Leave empty to crawl everything.", "default": "", "prefill": "", "editor": "textfield" }, "includeUrlGlobs": { "title": "Include URL Patterns (globs)", "type": "array", "description": "Only crawl discovered URLs matching these glob patterns (e.g. https://example.com/docs/*). Leave empty to allow everything on the domain.", "editor": "globs", "default": [], "prefill": [] }, "excludeUrlGlobs": { "title": "Exclude URL Patterns (globs)", "type": "array", "description": "Skip discovered URLs matching any of these glob patterns (e.g. *login*, *.pdf).", "editor": "globs", "default": [], "prefill": [] }, "includeUrlRegexps": { "title": "Include URL Patterns (regex)", "type": "array", "description": "Only crawl discovered URLs matching at least one of these regular expressions. Combined with glob patterns (both must pass when both are set).", "editor": "stringList", "default": [], "prefill": [] }, "excludeUrlRegexps": { "title": "Exclude URL Patterns (regex)", "type": "array", "description": "Skip discovered URLs matching any of these regular expressions.", "editor": "stringList", "default": [], "prefill": [] }, "keepUrlFragments": { "title": "Keep URL Fragments", "type": "boolean", "description": "Treat URLs that differ only in #fragment as different pages. Enable for single-page sites that use fragments as routes.", "default": false, "prefill": false }, "ignoreCanonicalUrl": { "title": "Ignore canonical URL", "type": "boolean", "description": "When disabled, pages whose canonical URL points to a page that was already crawled are skipped as duplicates.", "default": false, "prefill": false }, "outputFormat": { "title": "Output Format", "type": "string", "description": "How page content is returned. Markdown keeps headings, lists, tables, and links — best for LLMs and RAG.", "editor": "select", "default": "markdown", "prefill": "markdown", "enum": [ "markdown", "text", "html" ], "enumTitles": [ "Markdown", "Plain text", "Cleaned HTML" ], "sectionCaption": "Content extraction", "sectionDescription": "How page content is cleaned and returned." }, "htmlTransformer": { "title": "HTML Transformer", "type": "string", "description": "How the main content is isolated from navigation, footers, ads, and cookie banners.", "editor": "select", "default": "auto", "prefill": "auto", "enum": [ "auto", "trafilatura", "readability", "basic", "none" ], "enumTitles": [ "Auto (recommended) — best content, boilerplate removed", "Trafilatura (article/text extraction)", "Readability (Mozilla article extractor)", "Basic cleanup (remove nav/header/footer only)", "No cleaning (keep the full page)" ] }, "includeImages": { "title": "Include Images in Markdown", "type": "boolean", "description": "Add image references to the markdown field. Image URLs are always collected in the imageUrls field.", "default": false, "prefill": false }, "chunkMarkdown": { "title": "Split Content into Chunks", "type": "boolean", "description": "Emit a chunks array per page — heading-aware Markdown pieces ready for embeddings. Code blocks and tables are never split, each piece carries its heading path and character count, and consecutive pieces overlap so context is kept at the boundaries.", "default": false, "prefill": false }, "chunkMaxChars": { "title": "Chunk Size (characters)", "type": "integer", "description": "Target size of each chunk when chunkMarkdown is on. Code blocks and tables stay whole even when larger than this.", "minimum": 200, "maximum": 10000, "default": 2000, "prefill": 2000 }, "chunkOverlapChars": { "title": "Chunk Overlap (characters)", "type": "integer", "description": "Approximate overlap between consecutive chunks when chunkMarkdown is on, so context is not lost at chunk boundaries.", "minimum": 0, "maximum": 1000, "default": 200, "prefill": 200 }, "keepElementsCssSelector": { "title": "Keep HTML Elements (CSS selector)", "type": "string", "description": "Keep only the elements matching this CSS selector (e.g. article, main .content) before extraction. Leave empty to use the whole page.", "default": "", "prefill": "", "editor": "textfield" }, "removeElementsCssSelector": { "title": "Remove HTML Elements (CSS selector)", "type": "string", "description": "Remove elements matching this CSS selector (e.g. .sidebar, .ad, nav) before extraction.", "default": "", "prefill": "", "editor": "textfield" }, "parsePdfText": { "title": "Extract PDF Text", "type": "boolean", "description": "When a crawled URL serves a PDF, extract its text into the text field as a file record instead of skipping it.", "default": true, "prefill": true }, "clickElementsCssSelector": { "title": "Click Elements (CSS selector)", "type": "string", "description": "Click elements matching this CSS selector (e.g. [aria-expanded=\"false\"]) before extraction to expand collapsed content.", "default": "", "prefill": "", "editor": "textfield", "sectionCaption": "Rendered pages", "sectionDescription": "Options for content that only appears after JavaScript runs." }, "takeScreenshots": { "title": "Take Page Screenshots", "type": "boolean", "description": "Render every stored page in the browser and capture a full-page JPEG (quality 80) of it, putting its public URL in the screenshot field. Screenshots are held in memory until pushed and add roughly 0.3-1 MB and a fraction of a second per page; screenshot capture always runs after the page is rendered.", "default": false, "prefill": false }, "customHttpHeaders": { "title": "Custom HTTP Headers", "type": "array", "description": "Extra request headers for all engines and discovery requests, as a list of {\"key\", \"value\"} pairs (e.g. an Authorization token or a specific User-Agent for sites that ask for one). Transport/framing and cookie headers (Host, Cookie, Content-Length, Connection, ...) are rejected so the actor stays cookieless.", "editor": "json", "items": { "type": "object", "properties": { "key": { "type": "string", "title": "Header name", "description": "HTTP header name, e.g. Authorization or Accept-Language." }, "value": { "type": "string", "title": "Header value", "description": "Header value sent with every request." } }, "required": [ "key", "value" ] }, "default": [], "prefill": [], "sectionCaption": "Advanced", "sectionDescription": "Request headers and run diagnostics." }, "storeSkippedUrls": { "title": "Store Skipped URLs", "type": "boolean", "description": "Save a SKIPPED_URLS record to the key-value store listing skipped URLs grouped by reason (robots, blocked, non-HTML, errors, duplicates).", "default": false, "prefill": false } }, "required": [ "startUrls" ] }, "output": { "actorOutputSchemaVersion": 1, "title": "Domain-Scoped Site Crawler for Markdown Datasets Output", "description": "One record per crawled page: Markdown/text/HTML content, title, description, author, dates, canonical URL, links, images, OpenGraph, JSON-LD, word count, approximate token count, optional RAG chunks, optional screenshot URL, and content hash.", "properties": { "pages": { "type": "string", "title": "Crawled pages", "description": "Dataset containing all crawled pages and extracted files.", "template": "{{links.apiDefaultDatasetUrl}}/items" } } }, "storages": { "dataset": { "actorSpecification": 1, "views": { "overview": { "title": "Crawled Pages", "transformation": { "fields": [ "url", "title", "depth", "wordCount", "httpStatusCode" ] }, "display": { "component": "table", "properties": { "url": { "label": "URL", "format": "text" }, "title": { "label": "Title", "format": "text" }, "depth": { "label": "Depth", "format": "number" }, "wordCount": { "label": "Words", "format": "number" }, "httpStatusCode": { "label": "Status", "format": "number" } } } } } } } }, "buildNumber": "1.1.1", "imageDigest": "225e3518a23d1cb1a1e4b97ccf18192e761797514fbc49cf920a64e4129ed8a8", "actVersion": { "sourceType": "SOURCE_FILES", "buildTag": "latest", "versionNumber": "1.1" }, "usage": { "ACTOR_COMPUTE_UNITS": 0.04945666666666667 } } }