{ "openapi": "3.0.1", "info": { "title": "Remove Duplicates from Any Apify Dataset - Jobs, Leads, Places", "description": "Scheduled scrapes keep returning the same jobs, leads and places. Feed any Apify dataset, CSV or JSON and get only unique rows: key-field and fuzzy near-duplicate matching, URL tracking cleanup, and optional memory so each run returns only NEW items. Streams 1M rows.", "version": "2.0", "x-build-id": "vXm1QQ0jRGQCo8HzH" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/that_red_bird~remove-duplicate-scraped-data/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-that_red_bird-remove-duplicate-scraped-data", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/that_red_bird~remove-duplicate-scraped-data/runs": { "post": { "operationId": "runs-sync-that_red_bird-remove-duplicate-scraped-data", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/that_red_bird~remove-duplicate-scraped-data/run-sync": { "post": { "operationId": "run-sync-that_red_bird-remove-duplicate-scraped-data", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Apify datasets", "type": "array", "description": "Datasets (pick from your storage or pass dataset IDs) to clean, for example the dataset of a job, Google Maps or product scraper run. Several datasets are merged and de-duplicated together, which removes duplicates across runs and sources. Streams page by page, so 1M rows work. Use the picker rather than typing an ID: it gives this Actor permission to read your dataset." }, "records": { "title": "Rows (JSON)", "type": "array", "description": "Optional: rows pasted directly as an array of objects. Combined with the datasets above. The default is a small job-board sample so an empty run shows real output.", "default": [ { "title": "Senior Data Engineer", "company": "Northwind Analytics", "location": "Austin, TX", "url": "https://www.linkedin.com/jobs/view/senior-data-engineer-at-northwind-analytics-3912345678?refId=abc&trackingId=xyz", "salary": "$140k-$170k", "postedAt": "2026-10-01", "source": "linkedin", "description": "Build and maintain batch and streaming pipelines on Spark and Airflow. Own data quality for the revenue analytics platform." }, { "title": "Senior Data Engineer", "company": "Northwind Analytics", "location": "Austin, TX", "url": "https://www.linkedin.com/jobs/view/3912345678/?utm_source=daily-scrape", "salary": "$140k-$170k", "postedAt": "2026-10-02", "source": "linkedin", "description": "Build and maintain batch and streaming pipelines on Spark and Airflow. Own data quality for the revenue analytics platform." }, { "title": "Senior Data Engineer", "company": "Northwind Analytics", "location": "Austin, TX", "url": "https://careers.northwind-analytics.com/jobs/4471", "source": "company site", "description": "Build and maintain batch and streaming pipelines on Spark and Airflow. Own data quality for the revenue analytics platform. Remote friendly." }, { "title": "Product Designer", "company": "Lumen Labs", "location": "Remote (US)", "url": "https://boards.greenhouse.io/lumenlabs/jobs/5501234?gh_src=abc", "salary": "$110k-$130k", "postedAt": "2026-09-29", "source": "greenhouse", "description": "Design end-to-end product experiences for our B2B analytics suite, from research to polished UI." }, { "title": "Product Designer", "company": "Lumen Labs", "location": "Remote (US)", "url": "https://boards.greenhouse.io/lumenlabs/jobs/5501234", "salary": "$110k-$130k", "postedAt": "2026-10-02", "source": "greenhouse", "description": "Design end-to-end product experiences for our B2B analytics suite, from research to polished UI." }, { "title": "Customer Success Manager", "company": "Brightpath", "location": "Denver, CO", "url": "https://www.indeed.com/viewjob?jk=a1b2c3d4e5f60718&utm_campaign=x", "salary": "$85k-$100k", "postedAt": "2026-10-01", "source": "indeed", "description": "Own onboarding and renewals for a book of mid-market SaaS accounts and be the voice of the customer." }, { "title": "Customer Success Manager", "company": "Brightpath", "location": "Denver, CO", "url": "https://www.indeed.com/viewjob?jk=a1b2c3d4e5f60718", "salary": "$85k-$100k", "postedAt": "2026-10-03", "source": "indeed", "description": "Own onboarding and renewals for a book of mid-market SaaS accounts and be the voice of the customer." }, { "title": "DevOps Engineer", "company": "Cobalt Systems", "location": "Chicago, IL", "url": "https://jobs.lever.co/cobalt/8f2e1c77-aaaa-bbbb-cccc-0123456789ab", "salary": "$130k-$150k", "postedAt": "2026-09-30", "source": "lever", "description": "Run our Kubernetes platform, improve CI/CD and own on-call tooling for a 40-engineer organisation." }, { "title": "Registered Nurse - ICU", "company": "St. Mary Health", "location": "Portland, OR", "url": "https://www.stmaryhealth.org/careers/rn-icu-2291", "salary": "$48-$62/hr", "postedAt": "2026-10-02", "source": "company site", "description": "Provide bedside care in a 24-bed ICU, 12-hour night shifts, sign-on bonus available." }, { "title": "Registered Nurse - ICU", "company": "St. Mary Health", "location": "Portland, OR", "url": "https://www.indeed.com/viewjob?jk=ff00112233445566", "salary": "$48-$62/hr", "postedAt": "2026-10-03", "source": "indeed", "description": "Provide bedside care in a 24-bed ICU, 12-hour night shifts, sign-on bonus available." }, { "title": "Marketing Coordinator", "company": "Harbor & Pine", "location": "Boston, MA", "url": "https://www.linkedin.com/jobs/view/3955550001/", "salary": "$62k-$72k", "postedAt": "2026-10-01", "source": "linkedin", "description": "Support campaign execution, manage the content calendar and report weekly on channel performance." }, { "title": "Backend Engineer (Go)", "company": "Quickstream", "location": "Remote", "url": "https://jobs.ashbyhq.com/quickstream/9c1d3a52-1111-2222-3333-444455556666", "salary": "$150k-$180k", "postedAt": "2026-10-02", "source": "ashby", "description": "Design low-latency payment APIs in Go, with strong ownership of reliability and observability." }, { "title": "Backend Engineer (Go)", "company": "Quickstream", "location": "Remote", "url": "https://www.linkedin.com/jobs/view/3955559999/?refId=1", "salary": "$150k-$180k", "postedAt": "2026-10-03", "source": "linkedin", "description": "Design low-latency payment APIs in Go, with strong ownership of reliability and observability." }, { "title": "Accountant", "company": "Fieldstone & Co", "location": "Atlanta, GA", "url": "https://www.indeed.com/viewjob?jk=1122334455667788", "salary": "$70k-$85k", "postedAt": "2026-10-01", "source": "indeed", "description": "Prepare monthly close, reconcile accounts and support audits for a growing professional services firm." }, { "title": "Sales Development Representative", "company": "Orbit CRM", "location": "Miami, FL", "url": "https://boards.greenhouse.io/orbitcrm/jobs/6600777", "salary": "$55k-$65k + commission", "postedAt": "2026-10-03", "source": "greenhouse", "description": "Prospect, qualify and book meetings for our enterprise account executives." } ] }, "sourceUrl": { "title": "CSV or JSON file URL", "type": "string", "description": "Optional: a public URL to a CSV, JSON or NDJSON file. Leave empty to skip.", "default": "" }, "csvText": { "title": "Pasted CSV text", "type": "string", "description": "Optional: CSV text with a header row (comma, semicolon or tab separated). Leave empty to skip.", "default": "" }, "mode": { "title": "How to find duplicates", "enum": [ "keys", "wholeRecord", "fuzzy" ], "type": "string", "description": "keys = rows are duplicates when their key fields match (fast, exact, the default). wholeRecord = only fully identical rows. fuzzy = key matching PLUS near-duplicate text matching (reworded or lightly edited copies).", "default": "keys" }, "preset": { "title": "Key preset", "enum": [ "auto", "jobs", "places", "products", "leads", "posts" ], "type": "string", "description": "Which columns identify a row when \"Key fields\" is empty. Auto-detect looks at your columns and picks jobs, places, products, leads or posts keys. For example jobs: same url OR same title + company + location.", "default": "auto" }, "keyFields": { "title": "Key fields (custom)", "type": "array", "description": "Optional override. One entry per key: a column name (\"url\"), several alternatives (\"url|link\"), or a combination joined with + (\"title+company+location\"). A row is a duplicate if ANY entry matches an earlier row. Column names are case-insensitive and nested paths like address.city work.", "default": [], "items": { "type": "string" } }, "fuzzyFields": { "title": "Text fields for fuzzy matching", "type": "array", "description": "Fuzzy mode only. Columns whose text is compared for near-duplicates. Leave empty to auto-pick the longest text columns (ids, urls, dates and prices are ignored).", "default": [], "items": { "type": "string" } }, "similarityThreshold": { "title": "Fuzzy similarity threshold (0-100)", "minimum": 30, "maximum": 100, "type": "integer", "description": "Fuzzy mode only. Rows whose text is at least this similar (Jaccard of word or character shingles) are treated as duplicates. 85 catches re-posts and light rewording; raise it to be stricter.", "default": 85 }, "keepPolicy": { "title": "Which copy to keep", "enum": [ "first", "last", "richest" ], "type": "string", "description": "first = the earliest row of each duplicate group. last = the most recent row. richest = the row with the most filled-in fields.", "default": "first" }, "outputMode": { "title": "Dataset contents", "enum": [ "kept", "removed", "all" ], "type": "string", "description": "kept = only the unique rows, unchanged (ready to use). removed = only the removed duplicates with the reason and which row they duplicate. all = both, each tagged with _dedupeStatus. A REMOVED_DUPLICATES summary and run SUMMARY are always saved to the key-value store.", "default": "kept" }, "rememberAcrossRuns": { "title": "Return only NEW rows across runs", "type": "boolean", "description": "Turn on for scheduled scrapes: the Actor remembers the keys of rows it already delivered (in a named key-value store) and returns only rows it has not seen before. Use the same \"Memory name\" on every scheduled run.", "default": false }, "memoryName": { "title": "Memory name", "type": "string", "description": "Name of the remembered key set (letters, digits, dashes). Use a different name per scraper or search so their histories do not mix.", "default": "default" }, "caseSensitive": { "title": "Case-sensitive keys", "type": "boolean", "description": "When off, \"Acme Inc\" and \"ACME INC\" produce the same key. Accents and extra spaces are always ignored.", "default": false }, "normalizeUrls": { "title": "Clean URLs before comparing", "type": "boolean", "description": "Strip tracking parameters (utm_*, fbclid, gclid, refId, trackingId...), \"www.\", trailing slashes and, for LinkedIn job links, reduce to the job id, so the same posting with different tracking junk matches.", "default": true }, "ignoreFields": { "title": "Ignore fields (whole-record mode)", "type": "array", "description": "Whole-record mode only. Columns left out of the comparison, e.g. scrapedAt or crawlId, which differ on every run. Fields starting with \"_\" are always ignored.", "default": [], "items": { "type": "string" } }, "shingleSize": { "title": "Shingle size (fuzzy, advanced)", "minimum": 2, "maximum": 8, "type": "integer", "description": "Fuzzy mode only. Length of the word groups (long text) or character groups (short text) used to compare texts. 3 works well for most data.", "default": 3 }, "maxRemovedReport": { "title": "Max rows in removal report", "minimum": 0, "maximum": 200000, "type": "integer", "description": "How many removed rows are listed in the REMOVED_DUPLICATES key-value record (row number, which row it duplicates, why). Does not limit the dataset.", "default": 10000 }, "maxMemoryKeys": { "title": "Max remembered keys", "minimum": 1000, "maximum": 20000000, "type": "integer", "description": "Cap for the cross-run memory. When exceeded, the oldest remembered keys are dropped.", "default": 3000000 }, "maxRecords": { "title": "Max rows to process", "minimum": 1, "maximum": 10000000, "type": "integer", "description": "Upper limit of rows read from all sources. Memory use is roughly 100 bytes per row (about 260 with fuzzy mode), so give the run 2-4 GB for a million rows.", "default": 2000000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }