{ "data": { "id": "ajANmalEbfWPogs9n", "actId": "shC9wAKWFYbt1zZYs", "userId": "zq8lKbpLzTjGDayPC", "startedAt": "2026-10-30T01:07:50.506Z", "finishedAt": "2026-10-30T01:07:54.553Z", "status": "SUCCEEDED", "meta": { "origin": "CLI" }, "stats": { "durationMillis": 3932, "runTimeSecs": 3.932, "computeUnits": 0.004368888888888889, "imageSizeBytes": 318903808 }, "options": { "useCache": true, "betaPackages": false, "memoryMbytes": 4096, "diskMbytes": 8192 }, "createdByOrganizationMemberUserId": "GK9YjZRyQZVzOdoH5", "exitCode": 0, "inputSchema": "{\n \"title\": \"Hacker News Scraper Input\",\n \"description\": \"Choose what to collect from Hacker News: live feeds, historical front pages, search results, specific items or users, or the monthly Who is hiring? threads.\",\n \"type\": \"object\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"mode\": {\n \"title\": \"Mode\",\n \"type\": \"string\",\n \"description\": \"What to scrape. Feeds walk the live Hacker News lists. `item` looks up specific IDs, `user` loads profiles, `search` queries Hacker News history, `frontPage` reads a historical day, and `whoIsHiring` parses monthly hiring threads.\",\n \"editor\": \"select\",\n \"enum\": [\n \"topStories\",\n \"newStories\",\n \"bestStories\",\n \"askStories\",\n \"showStories\",\n \"jobStories\",\n \"item\",\n \"user\",\n \"search\",\n \"frontPage\",\n \"whoIsHiring\"\n ],\n \"enumTitles\": [\n \"Top stories\",\n \"New stories\",\n \"Best stories\",\n \"Ask HN\",\n \"Show HN\",\n \"Jobs\",\n \"Items by ID\",\n \"User profiles\",\n \"Search\",\n \"Historical front page\",\n \"Who is hiring?\"\n ],\n \"default\": \"topStories\",\n \"prefill\": \"topStories\"\n },\n \"itemIds\": {\n \"title\": \"Item IDs (mode=item)\",\n \"type\": \"array\",\n \"description\": \"Hacker News item IDs to look up, as numbers or numeric strings. Stories, comments, jobs, polls, and poll options are all accepted.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"usernames\": {\n \"title\": \"Usernames (mode=user)\",\n \"type\": \"array\",\n \"description\": \"Hacker News usernames whose profiles and, optionally, submissions you want.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"searchQueries\": {\n \"title\": \"Search queries (mode=search)\",\n \"type\": \"array\",\n \"description\": \"One or more keyword queries. Every query runs independently and the results are merged without duplicates. Leave empty to browse by tags, author, or story instead.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"searchSort\": {\n \"title\": \"Search order\",\n \"type\": \"string\",\n \"description\": \"`relevance` ranks by keyword match. `date` returns newest first and is more reproducible.\",\n \"editor\": \"select\",\n \"enum\": [\n \"relevance\",\n \"date\"\n ],\n \"enumTitles\": [\n \"Most relevant\",\n \"Newest first\"\n ],\n \"default\": \"relevance\",\n \"prefill\": \"relevance\"\n },\n \"contentTypes\": {\n \"title\": \"Search content types\",\n \"type\": \"array\",\n \"description\": \"Which Hacker News record types search should return. Selecting several types returns any of them. `front_page` covers only the current front page.\",\n \"editor\": \"select\",\n \"items\": {\n \"type\": \"string\",\n \"enum\": [\n \"story\",\n \"comment\",\n \"poll\",\n \"pollopt\",\n \"job\",\n \"show_hn\",\n \"ask_hn\",\n \"front_page\"\n ],\n \"enumTitles\": [\n \"Stories\",\n \"Comments\",\n \"Polls\",\n \"Poll options\",\n \"Jobs\",\n \"Show HN\",\n \"Ask HN\",\n \"Current front page\"\n ]\n },\n \"default\": [\n \"story\"\n ],\n \"prefill\": [\n \"story\"\n ]\n },\n \"author\": {\n \"title\": \"Author (mode=search)\",\n \"type\": \"string\",\n \"description\": \"Only return content written by this Hacker News username.\",\n \"editor\": \"textfield\"\n },\n \"storyId\": {\n \"title\": \"Story ID (mode=search)\",\n \"type\": \"string\",\n \"description\": \"Only return content attached to this story ID. Useful with `contentTypes` set to `comment` to search inside one thread.\",\n \"editor\": \"textfield\"\n },\n \"exhaustiveSearch\": {\n \"title\": \"Search past 1,000 results\",\n \"type\": \"boolean\",\n \"description\": \"Hacker News search exposes at most 1,000 hits per query. When enabled, the actor splits the requested date range into smaller windows so deeper result sets can be collected. Relevance order then follows each date window rather than one global ranking.\",\n \"default\": false,\n \"prefill\": false\n },\n \"dateRangeFrom\": {\n \"title\": \"Date range from\",\n \"type\": \"string\",\n \"description\": \"Drop items posted before this date (UTC, `YYYY-MM-DD`). Empty means no lower bound.\",\n \"editor\": \"textfield\"\n },\n \"dateRangeTo\": {\n \"title\": \"Date range to\",\n \"type\": \"string\",\n \"description\": \"Drop items posted after this date (UTC, `YYYY-MM-DD`). Empty means no upper bound.\",\n \"editor\": \"textfield\"\n },\n \"minScore\": {\n \"title\": \"Minimum points\",\n \"type\": \"integer\",\n \"description\": \"Drop stories, jobs, and polls below this point score. Comments are unaffected because Hacker News does not publish comment scores.\",\n \"minimum\": 0,\n \"maximum\": 1000000\n },\n \"minCommentCount\": {\n \"title\": \"Minimum comment count\",\n \"type\": \"integer\",\n \"description\": \"Drop stories and polls with fewer than this many comments.\",\n \"minimum\": 0,\n \"maximum\": 1000000\n },\n \"domainAllowlist\": {\n \"title\": \"Domain allowlist\",\n \"type\": \"array\",\n \"description\": \"Keep only stories whose link host contains one of these text fragments. For example, `github.com` or `arxiv.org`.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"domainBlocklist\": {\n \"title\": \"Domain blocklist\",\n \"type\": \"array\",\n \"description\": \"Drop stories whose link host contains one of these text fragments. For example, `twitter.com` or `x.com`.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"excludeDeadOrDeleted\": {\n \"title\": \"Exclude dead or deleted content\",\n \"type\": \"boolean\",\n \"description\": \"Drop items marked dead or deleted by Hacker News. Turning this off uses the slower exact comment path so those records can be included.\",\n \"default\": true,\n \"prefill\": true\n },\n \"commentAuthorFilter\": {\n \"title\": \"Comment author filter\",\n \"type\": \"array\",\n \"description\": \"Only keep comments written by these usernames. Matching is case-insensitive and accepts partial names.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"commentSort\": {\n \"title\": \"Comment order\",\n \"type\": \"string\",\n \"description\": \"`rank` keeps Hacker News display order. `newest` orders sibling replies newest first.\",\n \"editor\": \"select\",\n \"enum\": [\n \"rank\",\n \"newest\"\n ],\n \"enumTitles\": [\n \"Hacker News rank\",\n \"Newest first\"\n ],\n \"default\": \"rank\",\n \"prefill\": \"rank\"\n },\n \"maxComments\": {\n \"title\": \"Comments per story\",\n \"type\": \"integer\",\n \"description\": \"How many comments to collect for each story. 0 skips comment collection. This is applied after a story passes your filters.\",\n \"default\": 0,\n \"prefill\": 0,\n \"minimum\": 0,\n \"maximum\": 5000\n },\n \"maxDepth\": {\n \"title\": \"Maximum reply depth\",\n \"type\": \"integer\",\n \"description\": \"Deepest reply level to keep when comments are collected. Root comments are depth 0.\",\n \"default\": 5,\n \"prefill\": 5,\n \"minimum\": 0,\n \"maximum\": 20\n },\n \"enableCommentHierarchy\": {\n \"title\": \"Nest comment replies\",\n \"type\": \"boolean\",\n \"description\": \"When true, each story carries a nested `comments` tree. When false, comments are separate rows and never contain duplicate reply rows.\",\n \"default\": false,\n \"prefill\": false\n },\n \"includeSubmissions\": {\n \"title\": \"Include user submissions\",\n \"type\": \"boolean\",\n \"description\": \"For mode `user`, also emit the user's recent stories and comments as normal records after the profile.\",\n \"default\": false,\n \"prefill\": false\n },\n \"submissionTypes\": {\n \"title\": \"User submission types\",\n \"type\": \"string\",\n \"description\": \"Which submissions to emit when `includeSubmissions` is enabled.\",\n \"editor\": \"select\",\n \"enum\": [\n \"all\",\n \"story\",\n \"comment\"\n ],\n \"enumTitles\": [\n \"Stories and comments\",\n \"Stories only\",\n \"Comments only\"\n ],\n \"default\": \"all\",\n \"prefill\": \"all\"\n },\n \"frontPageDate\": {\n \"title\": \"Front page date\",\n \"type\": \"string\",\n \"description\": \"Historical day to read in mode `frontPage`, as `YYYY-MM-DD`. Empty uses yesterday (UTC).\",\n \"editor\": \"textfield\"\n },\n \"hiringMonths\": {\n \"title\": \"Hiring months\",\n \"type\": \"integer\",\n \"description\": \"How many recent monthly Who is hiring? threads to parse. Ignored when specific thread IDs are supplied.\",\n \"default\": 1,\n \"prefill\": 1,\n \"minimum\": 1,\n \"maximum\": 12\n },\n \"hiringThreadIds\": {\n \"title\": \"Hiring thread IDs\",\n \"type\": \"array\",\n \"description\": \"Specific Who is hiring? thread IDs to parse. Overrides the automatic monthly selection.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"hiringKeywords\": {\n \"title\": \"Hiring keywords\",\n \"type\": \"array\",\n \"description\": \"Keep only job posts whose text, company, role, location, or tech stack contains one of these terms.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n },\n \"remoteOnly\": {\n \"title\": \"Remote jobs only\",\n \"type\": \"boolean\",\n \"description\": \"Keep only hiring posts that explicitly mention remote work.\",\n \"default\": false,\n \"prefill\": false\n },\n \"maxItems\": {\n \"title\": \"Maximum records\",\n \"type\": \"integer\",\n \"description\": \"Hard cap on emitted records across all types. Nested comments also count toward this cap.\",\n \"default\": 50,\n \"prefill\": 20,\n \"minimum\": 1,\n \"maximum\": 5000\n },\n \"includeAuthorProfiles\": {\n \"title\": \"Add author karma\",\n \"type\": \"boolean\",\n \"description\": \"Add `authorKarma` and `authorCreatedAt` to stories and comments. This costs one extra lookup per unique author.\",\n \"default\": false,\n \"prefill\": false\n },\n \"startUrls\": {\n \"title\": \"Start URLs\",\n \"type\": \"array\",\n \"description\": \"Hacker News links to read directly: item, user, historical front-page, and hiring-thread URLs. IDs and URLs are merged with the mode-specific lists.\",\n \"editor\": \"stringList\",\n \"default\": [],\n \"prefill\": []\n }\n },\n \"required\": [\n \"mode\"\n ]\n}", "readme": "# Hacker News Archive: Stories, Comments & Hiring\n\nHacker News is a living archive of technology discussion: front-page links, Ask HN and Show HN threads, job ads, polls, user karma, and years of comments. This actor collects all of it from the public Hacker News surfaces — live feeds, keyword search, historical front pages, individual items, user profiles, and the monthly “Who is hiring?” threads. No login, cookies, API key, or proxy setup is needed.\n\n## What it does\n\n- **11 modes** — six live feeds, item lookup, user profiles, keyword search, historical front pages, and monthly hiring threads.\n- **Complete comment trees** — flat rows or nested `comments`, with no duplicated replies and a correct `maxItems` cap.\n- **Deep search** — relevance or newest-first ordering, tags, author, story, date range, points, and comment filters.\n- **Search past 1,000 results** — adaptive date-window splitting keeps collecting when a query is larger than the search API’s per-query ceiling.\n- **Typed records** — stories, comments, jobs, polls with vote options, poll options, user profiles, and parsed job postings.\n- **Historical front pages** — read any day back to 2007 with its original ranking, score, author, and comment count.\n\n## Services\n\nThe actor reads three public Hacker News services:\n\n| Service | What it provides |\n|---|---|\n| Firebase Hacker News API | Live feeds, individual items, user profiles, scores, comment counts, and exact deleted/dead flags |\n| Algolia Hacker News Search | Keyword search, date ordering, tag and author filters, and fast comment-tree reads |\n| Hacker News HTML | Historical `/front?day=` pages and their original story rankings |\n\nAll three are reachable without credentials.\n\n## How it works\n\nPick a mode and the actor chooses the right source for it:\n\n- **Feeds** read the complete live list for the chosen section, then fetch item details and apply your filters. It keeps walking until the feed ends or `maxItems` is reached, so filtered-out stories do not silently shorten the result.\n- **Search** sends your query, tags, author, story, and date range to Hacker News search. With `exhaustiveSearch` enabled it splits the date range whenever a window would exceed 1,000 hits, then merges the windows without duplicates.\n- **Comments** use the fast threaded read first. When a thread is missing live replies, the actor checks the thread’s comment search and merges the gap. Asking for dead or deleted content switches to the exact live item walk.\n- **Historical front pages** read page after page of the chosen day, then deduplicate stories that appear twice.\n- **Who is hiring?** finds the requested number of monthly threads, reads their top-level posts, and extracts the common hiring fields without inventing missing details.\n\nEvery mode is filter-aware: a story is checked before its comments are fetched, so filtered-out stories cost nothing extra.\n\n## Input\n\n| Field | Type | Default | Description |\n|---|---|---|---|\n| `mode` | string | `topStories` | `topStories` / `newStories` / `bestStories` / `askStories` / `showStories` / `jobStories` / `item` / `user` / `search` / `frontPage` / `whoIsHiring` |\n| `itemIds` | array | `[]` | Item IDs for `mode=item` |\n| `usernames` | array | `[]` | Usernames for `mode=user` |\n| `searchQueries` | array | `[]` | Keywords for `mode=search`; several queries are merged |\n| `searchSort` | string | `relevance` | `relevance` or `date` |\n| `contentTypes` | array | `[\"story\"]` | `story`, `comment`, `poll`, `pollopt`, `job`, `show_hn`, `ask_hn`, `front_page` |\n| `author` | string | – | Restrict search to one username |\n| `storyId` | string | – | Restrict search to one story |\n| `exhaustiveSearch` | boolean | `false` | Date-slice past the 1,000-hit search ceiling |\n| `dateRangeFrom` | string | – | Earliest date (`YYYY-MM-DD`, UTC) |\n| `dateRangeTo` | string | – | Latest date (`YYYY-MM-DD`, UTC) |\n| `minScore` | integer | – | Minimum points for stories, jobs, and polls |\n| `minCommentCount` | integer | – | Minimum comment count for stories and polls |\n| `domainAllowlist` | array | `[]` | Keep only these link-host fragments |\n| `domainBlocklist` | array | `[]` | Drop these link-host fragments |\n| `excludeDeadOrDeleted` | boolean | `true` | Drop dead/deleted items |\n| `commentAuthorFilter` | array | `[]` | Keep only comments by these authors |\n| `commentSort` | string | `rank` | `rank` or `newest` sibling order |\n| `maxComments` | integer | `0` | Comments collected per story; 0 skips comments |\n| `maxDepth` | integer | `5` | Maximum reply depth; roots are depth 0 |\n| `enableCommentHierarchy` | boolean | `false` | Nested `comments` instead of flat comment rows |\n| `includeSubmissions` | boolean | `false` | Also emit a user’s submissions |\n| `submissionTypes` | string | `all` | `all`, `story`, or `comment` |\n| `frontPageDate` | string | – | Day for `mode=frontPage`; empty means yesterday |\n| `hiringMonths` | integer | `1` | Recent monthly hiring threads to parse |\n| `hiringThreadIds` | array | `[]` | Specific hiring threads, overriding the month count |\n| `hiringKeywords` | array | `[]` | Keep matching hiring posts |\n| `remoteOnly` | boolean | `false` | Keep hiring posts that mention remote work |\n| `maxItems` | integer | `50` | Total records emitted; nested comments count too |\n| `includeAuthorProfiles` | boolean | `false` | Add author karma and account creation date |\n| `startUrls` | array | `[]` | Item, user, front-page, or hiring-thread links |\n\nEmpty fields are omitted from the output — there are no nulls or blank strings.\n\n### Example: search with filters\n\n```json\n{\n \"mode\": \"search\",\n \"searchQueries\": [\"rust async runtime\", \"tokio\"],\n \"searchSort\": \"date\",\n \"contentTypes\": [\"story\", \"comment\"],\n \"dateRangeFrom\": \"2025-01-01\",\n \"minScore\": 50,\n \"maxItems\": 200\n}\n```\n\n### Example: full comment tree\n\n```json\n{\n \"mode\": \"item\",\n \"itemIds\": [\"49922569\"],\n \"maxComments\": 300,\n \"maxDepth\": 10,\n \"enableCommentHierarchy\": true,\n \"commentSort\": \"newest\"\n}\n```\n\n### Example: a historical front page\n\n```json\n{\n \"mode\": \"frontPage\",\n \"frontPageDate\": \"2024-01-01\",\n \"maxItems\": 500\n}\n```\n\n### Example: this month’s hiring posts\n\n```json\n{\n \"mode\": \"whoIsHiring\",\n \"hiringMonths\": 1,\n \"hiringKeywords\": [\"python\", \"backend\"],\n \"remoteOnly\": true,\n \"maxItems\": 100\n}\n```\n\n## Output per story, job, or poll\n\n- `id`, `hnUrl`, `type` (`story`, `job`, or `poll`), `title`\n- `url`, `domain` for link posts\n- `score`, `author`, `numComments`\n- `createdAt` (ISO-8601 UTC), `createdAtEpoch`, `ageHours`\n- `text` for Ask HN, Show HN, jobs, and polls\n- `kids` — top-level comment IDs\n- `rank` — feed, search, or front-page position\n- `frontPageDate` for historical rows\n- `pollOptions[]` — `id`, `text`, `votes`, `author`, `createdAt`, `hnUrl`\n- `fetchedCommentCount` when comments were read\n- `comments[]` — nested replies when hierarchy is enabled\n- `authorKarma`, `authorCreatedAt` when author enrichment is enabled\n- `dead` / `deleted` only when true\n- `recordType: \"story\"`, `scrapedAt`\n\n## Output per comment\n\n- `id`, `hnUrl`, `type`, `author`, `text`\n- `storyId`, `parentId`, `depth`\n- `createdAt`, `createdAtEpoch`, `ageHours`\n- `kids` — reply IDs\n- `storyTitle`, `storyUrl` when a search hit supplies them\n- `replies[]` in nested mode only\n- `dead` / `deleted` only when true\n- `recordType: \"comment\"`, `scrapedAt`\n\n## Output per user\n\n- `id`, `profileUrl`\n- `karma`, `createdAt`, `createdAtEpoch`, `about`\n- `submittedCount`, `submitted` (first 50 recent IDs)\n- `recordType: \"user\"`, `scrapedAt`\n\nWith `includeSubmissions`, normal story and comment rows follow the profile.\n\n## Output per poll option\n\n- `id`, `hnUrl`, `type: \"pollopt\"`, `text`, `votes`, `pollId`\n- `author`, `createdAt`, `createdAtEpoch`, `ageHours`\n- `recordType: \"pollOption\"`, `scrapedAt`\n\n## Output per hiring post\n\n- `id`, `hnUrl`, `threadId`, `threadTitle`, `threadMonth`\n- `author`, `company`, `role`, `location`, `remote`, `salary`\n- `techStack`, `visa`, `email`, `applyUrl`\n- `text`, `postedAt`\n- `recordType: \"jobPosting\"`, `scrapedAt`\n\nParsed hiring fields are omitted when the post does not state them.\n\n## Usage cost\n\nThere is no paid data source, proxy, or credential in this actor. Usage is ordinary Apify compute plus the records you choose to collect. The cheapest useful run is the default prefill: 20 top stories with comments and author enrichment off. Cost grows with `maxItems`, `maxComments`, `maxDepth`, multiple search queries, `exhaustiveSearch`, and `includeAuthorProfiles` because each adds fetches. Small `maxItems` values keep runs short and inexpensive.\n\n## FAQ\n\n**Do I need an account, cookie, or API key?** No. Every mode reads public Hacker News data.\n\n**Do I need a proxy?** No. The actor connects directly and needs no proxy configuration.\n\n**Why is a field missing from some records?** Hacker News does not store that field for every item. For example, self posts have text but no link URL, and many users have no “about” text. Missing fields are omitted instead of filled with a placeholder.\n\n**Why do comment scores never appear?** Hacker News does not publish per-comment scores through its feeds, search, or HTML. Stories, polls, jobs, and poll options do have scores.\n\n**Why did a search stop at 1,000 results?** Hacker News search returns at most 1,000 hits per query window. Turn on `exhaustiveSearch` to split the date range into smaller windows and collect deeper results.\n\n**Why is relevance order different when `exhaustiveSearch` is on?** The deep search collects several date windows. Each window is ranked by relevance, so the final list follows window order rather than one global relevance ordering. Use `searchSort: \"date\"` for a stable chronological order.\n\n**How fresh is the data?** Feeds and item lookups are live. Search and comment search can lag the live site by a short period, so brand-new comments or score changes may appear slightly later.\n\n**What is the difference between `maxItems` and `maxComments`?** `maxItems` caps every emitted record, including nested comments. `maxComments` caps how many comments are read for each story before depth filtering.\n\n**Can I read a historical best, Ask HN, or Show HN list?** No. Hacker News archives only the historical front page by date. Live Ask HN, Show HN, best, and job lists are available through their normal modes.\n\n**Is this affiliated with Hacker News or Y Combinator?** No. This is an independent actor that reads Hacker News’ public data.\n\n## Notes\n\n- All timestamps are UTC and ISO-8601, with a matching Unix epoch.\n- Records are snapshots: scores, comment counts, and comment trees can change after a run.\n- Flat comment output contains one row per comment and never repeats a reply inside another row.\n- Nested comment output is one story row; every reply carries its own type and scrape timestamp.\n- `item` returns a comment row for a comment ID, a story-style row for a story, job, or poll, and a poll-option row for a poll-option ID.\n- `startUrls` accepts item, user, historical front-page, and hiring-thread links alongside the mode-specific lists.\n- Historical front pages are deduplicated when a story appears twice on the same day.\n\n## Limitations\n\n- **No comment scores or flags.** Hacker News does not expose per-comment scores, moderation flags, or edit history publicly.\n- **Search ceiling.** Each Hacker News search window returns at most 1,000 hits. `exhaustiveSearch` works around this by date slicing, but a single second with more than 1,000 hits cannot be split further.\n- **Historical lists.** Only the front page is archived by day. Historical best, Ask HN, Show HN, and jobs pages are not available.\n- **Feed caps.** Hacker News itself limits the live feeds: top and new stories usually expose 500 IDs, best about 200, Ask HN 20, Show HN up to about 150, and jobs about 30. The actor reports when a feed is exhausted.\n- **Comment-tree lag.** The fast tree can trail the live thread by a short period, and deleted branches may be absent. The actor merges a comment-search fallback when live replies are missing; turning off `excludeDeadOrDeleted` uses the exact slower walk.\n- **Hiring parsing is heuristic.** Company, role, location, salary, tech stack, visa, email, and apply link are extracted from free-form posts. Fields the post does not state are omitted; no value is invented.\n- **Very large user histories.** A user can have tens of thousands of submissions. `maxItems` and `submissionTypes` keep the run bounded.\n- **Relevance is not deterministic.** Search relevance can shift between runs as the index changes; date order is stable.\n", "actorDefinition": { "actorSpecification": 1, "name": "hacker-news-story-comment-and-karma-scraper", "title": "Hacker News Archive: Stories, Comments & Hiring", "description": "Collect Hacker News stories, comments, jobs, polls, users, historical front pages, and monthly Who is hiring? threads. Search by tag, author, story, date, points, and comments, with complete trees and deep results.", "version": "2.0", "buildTag": "latest", "dockerfile": "./Dockerfile", "readme": "# Hacker News Archive: Stories, Comments & Hiring\n\nHacker News is a living archive of technology discussion: front-page links, Ask HN and Show HN threads, job ads, polls, user karma, and years of comments. This actor collects all of it from the public Hacker News surfaces — live feeds, keyword search, historical front pages, individual items, user profiles, and the monthly “Who is hiring?” threads. No login, cookies, API key, or proxy setup is needed.\n\n## What it does\n\n- **11 modes** — six live feeds, item lookup, user profiles, keyword search, historical front pages, and monthly hiring threads.\n- **Complete comment trees** — flat rows or nested `comments`, with no duplicated replies and a correct `maxItems` cap.\n- **Deep search** — relevance or newest-first ordering, tags, author, story, date range, points, and comment filters.\n- **Search past 1,000 results** — adaptive date-window splitting keeps collecting when a query is larger than the search API’s per-query ceiling.\n- **Typed records** — stories, comments, jobs, polls with vote options, poll options, user profiles, and parsed job postings.\n- **Historical front pages** — read any day back to 2007 with its original ranking, score, author, and comment count.\n\n## Services\n\nThe actor reads three public Hacker News services:\n\n| Service | What it provides |\n|---|---|\n| Firebase Hacker News API | Live feeds, individual items, user profiles, scores, comment counts, and exact deleted/dead flags |\n| Algolia Hacker News Search | Keyword search, date ordering, tag and author filters, and fast comment-tree reads |\n| Hacker News HTML | Historical `/front?day=` pages and their original story rankings |\n\nAll three are reachable without credentials.\n\n## How it works\n\nPick a mode and the actor chooses the right source for it:\n\n- **Feeds** read the complete live list for the chosen section, then fetch item details and apply your filters. It keeps walking until the feed ends or `maxItems` is reached, so filtered-out stories do not silently shorten the result.\n- **Search** sends your query, tags, author, story, and date range to Hacker News search. With `exhaustiveSearch` enabled it splits the date range whenever a window would exceed 1,000 hits, then merges the windows without duplicates.\n- **Comments** use the fast threaded read first. When a thread is missing live replies, the actor checks the thread’s comment search and merges the gap. Asking for dead or deleted content switches to the exact live item walk.\n- **Historical front pages** read page after page of the chosen day, then deduplicate stories that appear twice.\n- **Who is hiring?** finds the requested number of monthly threads, reads their top-level posts, and extracts the common hiring fields without inventing missing details.\n\nEvery mode is filter-aware: a story is checked before its comments are fetched, so filtered-out stories cost nothing extra.\n\n## Input\n\n| Field | Type | Default | Description |\n|---|---|---|---|\n| `mode` | string | `topStories` | `topStories` / `newStories` / `bestStories` / `askStories` / `showStories` / `jobStories` / `item` / `user` / `search` / `frontPage` / `whoIsHiring` |\n| `itemIds` | array | `[]` | Item IDs for `mode=item` |\n| `usernames` | array | `[]` | Usernames for `mode=user` |\n| `searchQueries` | array | `[]` | Keywords for `mode=search`; several queries are merged |\n| `searchSort` | string | `relevance` | `relevance` or `date` |\n| `contentTypes` | array | `[\"story\"]` | `story`, `comment`, `poll`, `pollopt`, `job`, `show_hn`, `ask_hn`, `front_page` |\n| `author` | string | – | Restrict search to one username |\n| `storyId` | string | – | Restrict search to one story |\n| `exhaustiveSearch` | boolean | `false` | Date-slice past the 1,000-hit search ceiling |\n| `dateRangeFrom` | string | – | Earliest date (`YYYY-MM-DD`, UTC) |\n| `dateRangeTo` | string | – | Latest date (`YYYY-MM-DD`, UTC) |\n| `minScore` | integer | – | Minimum points for stories, jobs, and polls |\n| `minCommentCount` | integer | – | Minimum comment count for stories and polls |\n| `domainAllowlist` | array | `[]` | Keep only these link-host fragments |\n| `domainBlocklist` | array | `[]` | Drop these link-host fragments |\n| `excludeDeadOrDeleted` | boolean | `true` | Drop dead/deleted items |\n| `commentAuthorFilter` | array | `[]` | Keep only comments by these authors |\n| `commentSort` | string | `rank` | `rank` or `newest` sibling order |\n| `maxComments` | integer | `0` | Comments collected per story; 0 skips comments |\n| `maxDepth` | integer | `5` | Maximum reply depth; roots are depth 0 |\n| `enableCommentHierarchy` | boolean | `false` | Nested `comments` instead of flat comment rows |\n| `includeSubmissions` | boolean | `false` | Also emit a user’s submissions |\n| `submissionTypes` | string | `all` | `all`, `story`, or `comment` |\n| `frontPageDate` | string | – | Day for `mode=frontPage`; empty means yesterday |\n| `hiringMonths` | integer | `1` | Recent monthly hiring threads to parse |\n| `hiringThreadIds` | array | `[]` | Specific hiring threads, overriding the month count |\n| `hiringKeywords` | array | `[]` | Keep matching hiring posts |\n| `remoteOnly` | boolean | `false` | Keep hiring posts that mention remote work |\n| `maxItems` | integer | `50` | Total records emitted; nested comments count too |\n| `includeAuthorProfiles` | boolean | `false` | Add author karma and account creation date |\n| `startUrls` | array | `[]` | Item, user, front-page, or hiring-thread links |\n\nEmpty fields are omitted from the output — there are no nulls or blank strings.\n\n### Example: search with filters\n\n```json\n{\n \"mode\": \"search\",\n \"searchQueries\": [\"rust async runtime\", \"tokio\"],\n \"searchSort\": \"date\",\n \"contentTypes\": [\"story\", \"comment\"],\n \"dateRangeFrom\": \"2025-01-01\",\n \"minScore\": 50,\n \"maxItems\": 200\n}\n```\n\n### Example: full comment tree\n\n```json\n{\n \"mode\": \"item\",\n \"itemIds\": [\"49922569\"],\n \"maxComments\": 300,\n \"maxDepth\": 10,\n \"enableCommentHierarchy\": true,\n \"commentSort\": \"newest\"\n}\n```\n\n### Example: a historical front page\n\n```json\n{\n \"mode\": \"frontPage\",\n \"frontPageDate\": \"2024-01-01\",\n \"maxItems\": 500\n}\n```\n\n### Example: this month’s hiring posts\n\n```json\n{\n \"mode\": \"whoIsHiring\",\n \"hiringMonths\": 1,\n \"hiringKeywords\": [\"python\", \"backend\"],\n \"remoteOnly\": true,\n \"maxItems\": 100\n}\n```\n\n## Output per story, job, or poll\n\n- `id`, `hnUrl`, `type` (`story`, `job`, or `poll`), `title`\n- `url`, `domain` for link posts\n- `score`, `author`, `numComments`\n- `createdAt` (ISO-8601 UTC), `createdAtEpoch`, `ageHours`\n- `text` for Ask HN, Show HN, jobs, and polls\n- `kids` — top-level comment IDs\n- `rank` — feed, search, or front-page position\n- `frontPageDate` for historical rows\n- `pollOptions[]` — `id`, `text`, `votes`, `author`, `createdAt`, `hnUrl`\n- `fetchedCommentCount` when comments were read\n- `comments[]` — nested replies when hierarchy is enabled\n- `authorKarma`, `authorCreatedAt` when author enrichment is enabled\n- `dead` / `deleted` only when true\n- `recordType: \"story\"`, `scrapedAt`\n\n## Output per comment\n\n- `id`, `hnUrl`, `type`, `author`, `text`\n- `storyId`, `parentId`, `depth`\n- `createdAt`, `createdAtEpoch`, `ageHours`\n- `kids` — reply IDs\n- `storyTitle`, `storyUrl` when a search hit supplies them\n- `replies[]` in nested mode only\n- `dead` / `deleted` only when true\n- `recordType: \"comment\"`, `scrapedAt`\n\n## Output per user\n\n- `id`, `profileUrl`\n- `karma`, `createdAt`, `createdAtEpoch`, `about`\n- `submittedCount`, `submitted` (first 50 recent IDs)\n- `recordType: \"user\"`, `scrapedAt`\n\nWith `includeSubmissions`, normal story and comment rows follow the profile.\n\n## Output per poll option\n\n- `id`, `hnUrl`, `type: \"pollopt\"`, `text`, `votes`, `pollId`\n- `author`, `createdAt`, `createdAtEpoch`, `ageHours`\n- `recordType: \"pollOption\"`, `scrapedAt`\n\n## Output per hiring post\n\n- `id`, `hnUrl`, `threadId`, `threadTitle`, `threadMonth`\n- `author`, `company`, `role`, `location`, `remote`, `salary`\n- `techStack`, `visa`, `email`, `applyUrl`\n- `text`, `postedAt`\n- `recordType: \"jobPosting\"`, `scrapedAt`\n\nParsed hiring fields are omitted when the post does not state them.\n\n## Usage cost\n\nThere is no paid data source, proxy, or credential in this actor. Usage is ordinary Apify compute plus the records you choose to collect. The cheapest useful run is the default prefill: 20 top stories with comments and author enrichment off. Cost grows with `maxItems`, `maxComments`, `maxDepth`, multiple search queries, `exhaustiveSearch`, and `includeAuthorProfiles` because each adds fetches. Small `maxItems` values keep runs short and inexpensive.\n\n## FAQ\n\n**Do I need an account, cookie, or API key?** No. Every mode reads public Hacker News data.\n\n**Do I need a proxy?** No. The actor connects directly and needs no proxy configuration.\n\n**Why is a field missing from some records?** Hacker News does not store that field for every item. For example, self posts have text but no link URL, and many users have no “about” text. Missing fields are omitted instead of filled with a placeholder.\n\n**Why do comment scores never appear?** Hacker News does not publish per-comment scores through its feeds, search, or HTML. Stories, polls, jobs, and poll options do have scores.\n\n**Why did a search stop at 1,000 results?** Hacker News search returns at most 1,000 hits per query window. Turn on `exhaustiveSearch` to split the date range into smaller windows and collect deeper results.\n\n**Why is relevance order different when `exhaustiveSearch` is on?** The deep search collects several date windows. Each window is ranked by relevance, so the final list follows window order rather than one global relevance ordering. Use `searchSort: \"date\"` for a stable chronological order.\n\n**How fresh is the data?** Feeds and item lookups are live. Search and comment search can lag the live site by a short period, so brand-new comments or score changes may appear slightly later.\n\n**What is the difference between `maxItems` and `maxComments`?** `maxItems` caps every emitted record, including nested comments. `maxComments` caps how many comments are read for each story before depth filtering.\n\n**Can I read a historical best, Ask HN, or Show HN list?** No. Hacker News archives only the historical front page by date. Live Ask HN, Show HN, best, and job lists are available through their normal modes.\n\n**Is this affiliated with Hacker News or Y Combinator?** No. This is an independent actor that reads Hacker News’ public data.\n\n## Notes\n\n- All timestamps are UTC and ISO-8601, with a matching Unix epoch.\n- Records are snapshots: scores, comment counts, and comment trees can change after a run.\n- Flat comment output contains one row per comment and never repeats a reply inside another row.\n- Nested comment output is one story row; every reply carries its own type and scrape timestamp.\n- `item` returns a comment row for a comment ID, a story-style row for a story, job, or poll, and a poll-option row for a poll-option ID.\n- `startUrls` accepts item, user, historical front-page, and hiring-thread links alongside the mode-specific lists.\n- Historical front pages are deduplicated when a story appears twice on the same day.\n\n## Limitations\n\n- **No comment scores or flags.** Hacker News does not expose per-comment scores, moderation flags, or edit history publicly.\n- **Search ceiling.** Each Hacker News search window returns at most 1,000 hits. `exhaustiveSearch` works around this by date slicing, but a single second with more than 1,000 hits cannot be split further.\n- **Historical lists.** Only the front page is archived by day. Historical best, Ask HN, Show HN, and jobs pages are not available.\n- **Feed caps.** Hacker News itself limits the live feeds: top and new stories usually expose 500 IDs, best about 200, Ask HN 20, Show HN up to about 150, and jobs about 30. The actor reports when a feed is exhausted.\n- **Comment-tree lag.** The fast tree can trail the live thread by a short period, and deleted branches may be absent. The actor merges a comment-search fallback when live replies are missing; turning off `excludeDeadOrDeleted` uses the exact slower walk.\n- **Hiring parsing is heuristic.** Company, role, location, salary, tech stack, visa, email, and apply link are extracted from free-form posts. Fields the post does not state are omitted; no value is invented.\n- **Very large user histories.** A user can have tens of thousands of submissions. `maxItems` and `submissionTypes` keep the run bounded.\n- **Relevance is not deterministic.** Search relevance can shift between runs as the index changes; date order is stable.\n", "input": { "title": "Hacker News Scraper Input", "description": "Choose what to collect from Hacker News: live feeds, historical front pages, search results, specific items or users, or the monthly Who is hiring? threads.", "type": "object", "schemaVersion": 1, "properties": { "mode": { "title": "Mode", "type": "string", "description": "What to scrape. Feeds walk the live Hacker News lists. `item` looks up specific IDs, `user` loads profiles, `search` queries Hacker News history, `frontPage` reads a historical day, and `whoIsHiring` parses monthly hiring threads.", "editor": "select", "enum": [ "topStories", "newStories", "bestStories", "askStories", "showStories", "jobStories", "item", "user", "search", "frontPage", "whoIsHiring" ], "enumTitles": [ "Top stories", "New stories", "Best stories", "Ask HN", "Show HN", "Jobs", "Items by ID", "User profiles", "Search", "Historical front page", "Who is hiring?" ], "default": "topStories", "prefill": "topStories" }, "itemIds": { "title": "Item IDs (mode=item)", "type": "array", "description": "Hacker News item IDs to look up, as numbers or numeric strings. Stories, comments, jobs, polls, and poll options are all accepted.", "editor": "stringList", "default": [], "prefill": [] }, "usernames": { "title": "Usernames (mode=user)", "type": "array", "description": "Hacker News usernames whose profiles and, optionally, submissions you want.", "editor": "stringList", "default": [], "prefill": [] }, "searchQueries": { "title": "Search queries (mode=search)", "type": "array", "description": "One or more keyword queries. Every query runs independently and the results are merged without duplicates. Leave empty to browse by tags, author, or story instead.", "editor": "stringList", "default": [], "prefill": [] }, "searchSort": { "title": "Search order", "type": "string", "description": "`relevance` ranks by keyword match. `date` returns newest first and is more reproducible.", "editor": "select", "enum": [ "relevance", "date" ], "enumTitles": [ "Most relevant", "Newest first" ], "default": "relevance", "prefill": "relevance" }, "contentTypes": { "title": "Search content types", "type": "array", "description": "Which Hacker News record types search should return. Selecting several types returns any of them. `front_page` covers only the current front page.", "editor": "select", "items": { "type": "string", "enum": [ "story", "comment", "poll", "pollopt", "job", "show_hn", "ask_hn", "front_page" ], "enumTitles": [ "Stories", "Comments", "Polls", "Poll options", "Jobs", "Show HN", "Ask HN", "Current front page" ] }, "default": [ "story" ], "prefill": [ "story" ] }, "author": { "title": "Author (mode=search)", "type": "string", "description": "Only return content written by this Hacker News username.", "editor": "textfield" }, "storyId": { "title": "Story ID (mode=search)", "type": "string", "description": "Only return content attached to this story ID. Useful with `contentTypes` set to `comment` to search inside one thread.", "editor": "textfield" }, "exhaustiveSearch": { "title": "Search past 1,000 results", "type": "boolean", "description": "Hacker News search exposes at most 1,000 hits per query. When enabled, the actor splits the requested date range into smaller windows so deeper result sets can be collected. Relevance order then follows each date window rather than one global ranking.", "default": false, "prefill": false }, "dateRangeFrom": { "title": "Date range from", "type": "string", "description": "Drop items posted before this date (UTC, `YYYY-MM-DD`). Empty means no lower bound.", "editor": "textfield" }, "dateRangeTo": { "title": "Date range to", "type": "string", "description": "Drop items posted after this date (UTC, `YYYY-MM-DD`). Empty means no upper bound.", "editor": "textfield" }, "minScore": { "title": "Minimum points", "type": "integer", "description": "Drop stories, jobs, and polls below this point score. Comments are unaffected because Hacker News does not publish comment scores.", "minimum": 0, "maximum": 1000000 }, "minCommentCount": { "title": "Minimum comment count", "type": "integer", "description": "Drop stories and polls with fewer than this many comments.", "minimum": 0, "maximum": 1000000 }, "domainAllowlist": { "title": "Domain allowlist", "type": "array", "description": "Keep only stories whose link host contains one of these text fragments. For example, `github.com` or `arxiv.org`.", "editor": "stringList", "default": [], "prefill": [] }, "domainBlocklist": { "title": "Domain blocklist", "type": "array", "description": "Drop stories whose link host contains one of these text fragments. For example, `twitter.com` or `x.com`.", "editor": "stringList", "default": [], "prefill": [] }, "excludeDeadOrDeleted": { "title": "Exclude dead or deleted content", "type": "boolean", "description": "Drop items marked dead or deleted by Hacker News. Turning this off uses the slower exact comment path so those records can be included.", "default": true, "prefill": true }, "commentAuthorFilter": { "title": "Comment author filter", "type": "array", "description": "Only keep comments written by these usernames. Matching is case-insensitive and accepts partial names.", "editor": "stringList", "default": [], "prefill": [] }, "commentSort": { "title": "Comment order", "type": "string", "description": "`rank` keeps Hacker News display order. `newest` orders sibling replies newest first.", "editor": "select", "enum": [ "rank", "newest" ], "enumTitles": [ "Hacker News rank", "Newest first" ], "default": "rank", "prefill": "rank" }, "maxComments": { "title": "Comments per story", "type": "integer", "description": "How many comments to collect for each story. 0 skips comment collection. This is applied after a story passes your filters.", "default": 0, "prefill": 0, "minimum": 0, "maximum": 5000 }, "maxDepth": { "title": "Maximum reply depth", "type": "integer", "description": "Deepest reply level to keep when comments are collected. Root comments are depth 0.", "default": 5, "prefill": 5, "minimum": 0, "maximum": 20 }, "enableCommentHierarchy": { "title": "Nest comment replies", "type": "boolean", "description": "When true, each story carries a nested `comments` tree. When false, comments are separate rows and never contain duplicate reply rows.", "default": false, "prefill": false }, "includeSubmissions": { "title": "Include user submissions", "type": "boolean", "description": "For mode `user`, also emit the user's recent stories and comments as normal records after the profile.", "default": false, "prefill": false }, "submissionTypes": { "title": "User submission types", "type": "string", "description": "Which submissions to emit when `includeSubmissions` is enabled.", "editor": "select", "enum": [ "all", "story", "comment" ], "enumTitles": [ "Stories and comments", "Stories only", "Comments only" ], "default": "all", "prefill": "all" }, "frontPageDate": { "title": "Front page date", "type": "string", "description": "Historical day to read in mode `frontPage`, as `YYYY-MM-DD`. Empty uses yesterday (UTC).", "editor": "textfield" }, "hiringMonths": { "title": "Hiring months", "type": "integer", "description": "How many recent monthly Who is hiring? threads to parse. Ignored when specific thread IDs are supplied.", "default": 1, "prefill": 1, "minimum": 1, "maximum": 12 }, "hiringThreadIds": { "title": "Hiring thread IDs", "type": "array", "description": "Specific Who is hiring? thread IDs to parse. Overrides the automatic monthly selection.", "editor": "stringList", "default": [], "prefill": [] }, "hiringKeywords": { "title": "Hiring keywords", "type": "array", "description": "Keep only job posts whose text, company, role, location, or tech stack contains one of these terms.", "editor": "stringList", "default": [], "prefill": [] }, "remoteOnly": { "title": "Remote jobs only", "type": "boolean", "description": "Keep only hiring posts that explicitly mention remote work.", "default": false, "prefill": false }, "maxItems": { "title": "Maximum records", "type": "integer", "description": "Hard cap on emitted records across all types. Nested comments also count toward this cap.", "default": 50, "prefill": 20, "minimum": 1, "maximum": 5000 }, "includeAuthorProfiles": { "title": "Add author karma", "type": "boolean", "description": "Add `authorKarma` and `authorCreatedAt` to stories and comments. This costs one extra lookup per unique author.", "default": false, "prefill": false }, "startUrls": { "title": "Start URLs", "type": "array", "description": "Hacker News links to read directly: item, user, historical front-page, and hiring-thread URLs. IDs and URLs are merged with the mode-specific lists.", "editor": "stringList", "default": [], "prefill": [] } }, "required": [ "mode" ] }, "output": { "actorOutputSchemaVersion": 1, "title": "Hacker News Archive: Stories, Comments & Hiring Output", "description": "Stories, comments, jobs, polls, poll options, user profiles, and parsed Who is hiring? posts from Hacker News.", "properties": { "items": { "type": "string", "title": "Hacker News records", "description": "Dataset containing every scraped record. `recordType` identifies stories, comments, users, poll options, and hiring posts. Story rows include id, title, url, domain, score, author, numComments, rank, timestamps, text, pollOptions, and optional nested comments. Comment rows include id, storyId, parentId, depth, author, text, and story context. User rows include karma, account age, about text, and submission counts.", "template": "{{links.apiDefaultDatasetUrl}}/items" } } }, "storages": { "dataset": { "actorSpecification": 1, "views": { "overview": { "title": "Hacker News", "transformation": { "fields": [ "recordType", "title", "company", "role", "url", "score", "author", "numComments", "createdAt", "domain", "frontPageDate" ] }, "display": { "component": "table", "properties": {} } } } } }, "environmentVariables": {} }, "buildNumber": "2.0.1", "imageDigest": "031e1607c9c6302fdeaf372a387bfe8f0ca04b0541c5a420342bf5c7a77df076", "actVersion": { "sourceType": "SOURCE_FILES", "buildTag": "latest", "versionNumber": "2.0" }, "usage": { "ACTOR_COMPUTE_UNITS": 0.004368888888888889 } } }