{ "name": "RSS feed ingestion \u2014 BBC + Hacker News (manual)", "flow": [ { "id": 1, "module": "apify:runActorNew", "version": 1, "parameters": {}, "mapper": { "actor_search_type": "store", "actorId": "LR87biSfc03YzjqVZ", "runSync": true, "timeout": 90, "maxTotalChargeUsd": 1, "inputBodyLR87biSfc03YzjqVZ": "{\n \"feedUrls\": [\n \"https://feeds.bbci.co.uk/news/rss.xml\",\n \"https://news.ycombinator.com/rss\"\n ],\n \"maxItemsPerFeed\": 5,\n \"includeContent\": true\n}" }, "metadata": { "designer": { "x": 0, "y": 0, "name": "Run once \u2014 $1 maximum, 90 seconds" }, "parameters": [ { "name": "__IMTCONN__", "type": "account:apify2,apify", "label": "Connection", "required": true } ], "expect": [ { "name": "actor_search_type", "type": "select", "label": "Search Actors from", "required": true, "validate": { "enum": [ "user", "store" ] } }, { "name": "runSync", "type": "boolean", "label": "Run synchronously", "required": true }, { "name": "build", "type": "text", "label": "Build" }, { "name": "timeout", "type": "uinteger", "label": "Timeout" }, { "name": "memory", "type": "select", "label": "Memory", "validate": { "enum": [ 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768 ] } }, { "name": "actorId", "type": "select", "label": "Actor", "required": true }, { "name": "inputBodyLR87biSfc03YzjqVZ", "type": "text", "label": "Input JSON", "required": true }, { "name": "maxTotalChargeUsd", "type": "number", "label": "Maximum cost per run" }, { "name": "maxItems", "type": "hidden", "label": "Maximum charged results" } ], "restore": { "expect": { "actor_search_type": { "label": "Apify Store" }, "actorId": { "mode": "chose", "label": "technicaldost/rss-feed-scraper" }, "runSync": { "mode": "chose" } } } }, "onerror": [ { "id": 101, "module": "builtin:Rollback", "version": 1, "parameters": {}, "mapper": {}, "metadata": { "designer": { "x": 300, "y": 300, "name": "Stop; do not retry paid start" } } } ] }, { "id": 2, "module": "builtin:BasicRouter", "version": 1, "parameters": {}, "mapper": null, "metadata": { "designer": { "x": 300, "y": 0, "name": "Require completed success" } }, "routes": [ { "flow": [ { "id": 9, "module": "json:ParseJSON", "version": 1, "parameters": {}, "mapper": { "json": "ACTOR_NOT_SUCCEEDED: status={{ifempty(1.status; \"MISSING\")}} run={{1.id}}. Inspect the existing run in Apify; do not automatically start another paid run." }, "metadata": { "designer": { "x": 600, "y": 500, "name": "ERROR \u2014 inspect run; no automatic retry" } }, "onerror": [ { "id": 109, "module": "builtin:Rollback", "version": 1, "parameters": {}, "mapper": {}, "metadata": { "designer": { "x": 900, "y": 660, "name": "Stop; do not retry paid start" } } } ], "filter": { "name": "Every other run status", "conditions": [ [ { "a": "{{ifempty(1.status; \"MISSING\")}}", "o": "text:notequal", "b": "SUCCEEDED" } ] ] } } ] }, { "flow": [ { "id": 3, "module": "apify:fetchDatasetItems", "version": 1, "parameters": {}, "mapper": { "datasetId": "{{1.defaultDatasetId}}", "type": "none", "format": "json", "limit": "100", "offset": "0" }, "metadata": { "designer": { "x": 600, "y": 0, "name": "Read bounded dataset" }, "parameters": [ { "name": "__IMTCONN__", "type": "account:apify2,apify", "label": "Connection", "required": true } ], "expect": [ { "name": "datasetId", "type": "text", "label": "Dataset ID", "required": true }, { "name": "type", "type": "select", "label": "Data transformation", "required": true, "validate": { "enum": [ "clean", "simplified", "none" ] } }, { "name": "format", "type": "select", "label": "Format", "required": true, "validate": { "enum": [ "json", "csv", "html", "xml", "rss", "xlsx" ] } }, { "name": "limit", "type": "uinteger", "label": "Limit", "validate": { "max": 100000, "min": 1 } }, { "name": "offset", "type": "uinteger", "label": "Offset", "validate": { "min": 0 } }, { "name": "fields", "type": "text", "label": "Fields" }, { "name": "omit", "type": "text", "label": "Omit Fields" } ] }, "filter": { "name": "Actor must have SUCCEEDED", "conditions": [ [ { "a": "{{ifempty(1.status; \"MISSING\")}}", "o": "text:equal", "b": "SUCCEEDED" } ] ] }, "onerror": [ { "id": 103, "module": "builtin:Rollback", "version": 1, "parameters": {}, "mapper": {}, "metadata": { "designer": { "x": 900, "y": -250, "name": "Stop; do not retry paid start" } } } ] }, { "id": 4, "module": "builtin:BasicAggregator", "version": 1, "parameters": { "feeder": 3 }, "mapper": { "type": "{{3.type}}", "feedTitle": "{{3.feedTitle}}", "feedUrl": "{{3.feedUrl}}", "title": "{{3.title}}", "link": "{{3.link}}", "pubDate": "{{3.pubDate}}", "author": "{{3.author}}", "categories": "{{3.categories}}", "guid": "{{3.guid}}", "summary": "{{3.summary}}", "content": "{{3.content}}", "enclosure": "{{3.enclosure}}", "mediaContent": "{{3.mediaContent}}", "thumbnail": "{{3.thumbnail}}", "scrapedAt": "{{3.scrapedAt}}", "error": "{{3.error}}" }, "metadata": { "designer": { "x": 900, "y": 0, "name": "Gather all rows before validation" }, "restore": { "extra": { "feeder": { "label": "Apify - Get Dataset Items [3]" }, "target": { "label": "Custom" } } } }, "flags": { "stopIfEmpty": false } }, { "id": 5, "module": "builtin:BasicRouter", "version": 1, "parameters": {}, "mapper": null, "metadata": { "designer": { "x": 1200, "y": 0, "name": "Validate dataset before output" } }, "routes": [ { "flow": [ { "id": 6, "module": "json:ParseJSON", "version": 1, "parameters": {}, "mapper": { "json": "RSS_DATASET_ERROR: errors={{length(map(4.array; \"type\"; \"type\"; \"error\"))}} feed_items={{length(map(4.array; \"type\"; \"type\"; \"feed_item\"))}} rows={{length(4.array)}}. Review dataset {{1.defaultDatasetId}}. No results delivered." }, "metadata": { "designer": { "x": 1500, "y": 300, "name": "ERROR \u2014 inspect run; no automatic retry" } }, "onerror": [ { "id": 106, "module": "builtin:Rollback", "version": 1, "parameters": {}, "mapper": {}, "metadata": { "designer": { "x": 1800, "y": 460, "name": "Stop; do not retry paid start" } } } ], "filter": { "name": "Feed errors OR no items OR possible truncation", "conditions": [ [ { "a": "{{length(map(4.array; \"type\"; \"type\"; \"error\"))}}", "o": "number:greater", "b": "0" } ], [ { "a": "{{length(map(4.array; \"type\"; \"type\"; \"feed_item\"))}}", "o": "number:equal", "b": "0" } ], [ { "a": "{{length(4.array)}}", "o": "number:greater", "b": "99" } ] ] } } ] }, { "flow": [ { "id": 7, "module": "builtin:BasicFeeder", "version": 1, "parameters": {}, "mapper": { "array": "{{4.array}}" }, "metadata": { "designer": { "x": 1500, "y": 0, "name": "Iterate validated dataset" } }, "filter": { "name": "No feed errors; nonempty; below row limit", "conditions": [ [ { "a": "{{length(map(4.array; \"type\"; \"type\"; \"error\"))}}", "o": "number:equal", "b": "0" }, { "a": "{{length(map(4.array; \"type\"; \"type\"; \"feed_item\"))}}", "o": "number:greater", "b": "0" }, { "a": "{{length(4.array)}}", "o": "number:less", "b": "100" } ] ] } }, { "id": 8, "module": "util:SetVariables", "version": 1, "parameters": {}, "mapper": { "variables": [ { "name": "type", "value": "{{7.type}}" }, { "name": "feedTitle", "value": "{{7.feedTitle}}" }, { "name": "feedUrl", "value": "{{7.feedUrl}}" }, { "name": "title", "value": "{{7.title}}" }, { "name": "link", "value": "{{7.link}}" }, { "name": "pubDate", "value": "{{7.pubDate}}" }, { "name": "author", "value": "{{7.author}}" }, { "name": "categories", "value": "{{7.categories}}" }, { "name": "guid", "value": "{{7.guid}}" }, { "name": "summary", "value": "{{7.summary}}" }, { "name": "content", "value": "{{7.content}}" }, { "name": "enclosure", "value": "{{7.enclosure}}" }, { "name": "mediaContent", "value": "{{7.mediaContent}}" }, { "name": "thumbnail", "value": "{{7.thumbnail}}" }, { "name": "scrapedAt", "value": "{{7.scrapedAt}}" } ], "scope": "roundtrip" }, "metadata": { "designer": { "x": 1800, "y": 0, "name": "Output: feed_item fields only" }, "interface": [ { "name": "type", "type": "any", "label": "type" }, { "name": "feedTitle", "type": "any", "label": "feedTitle" }, { "name": "feedUrl", "type": "any", "label": "feedUrl" }, { "name": "title", "type": "any", "label": "title" }, { "name": "link", "type": "any", "label": "link" }, { "name": "pubDate", "type": "any", "label": "pubDate" }, { "name": "author", "type": "any", "label": "author" }, { "name": "categories", "type": "any", "label": "categories" }, { "name": "guid", "type": "any", "label": "guid" }, { "name": "summary", "type": "any", "label": "summary" }, { "name": "content", "type": "any", "label": "content" }, { "name": "enclosure", "type": "any", "label": "enclosure" }, { "name": "mediaContent", "type": "any", "label": "mediaContent" }, { "name": "thumbnail", "type": "any", "label": "thumbnail" }, { "name": "scrapedAt", "type": "any", "label": "scrapedAt" } ] }, "filter": { "name": "Only feed_item rows", "conditions": [ [ { "a": "{{7.type}}", "o": "text:equal", "b": "feed_item" } ] ] } } ] } ] } ] } ] } ], "metadata": { "instant": false, "version": 1, "scenario": { "roundtrips": 1, "maxErrors": 1, "autoCommit": true, "autoCommitTriggerLast": true, "sequential": true, "slots": null, "confidential": false, "dataloss": false, "dlq": false, "freshVariables": false }, "designer": { "orphans": [] }, "notes": [] } }