{ "openapi": "3.0.1", "info": { "title": "Remove Duplicates: Deduplicate Datasets, CSV & Excel", "description": "Remove duplicates from Apify datasets, pasted JSON, CSV, Excel and JSONL files or text lines. Deduplicate by chosen fields or whole rows, exact or fuzzy, with email, URL, phone and domain normalisers. Keep first, last or most complete, or merge. Get clean rows and a duplicates report.", "version": "0.1", "x-build-id": "uqI5vDE1h0IYZZYzI" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/data-done-right~remove-duplicates/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-data-done-right-remove-duplicates", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/data-done-right~remove-duplicates/runs": { "post": { "operationId": "runs-sync-data-done-right-remove-duplicates", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/data-done-right~remove-duplicates/run-sync": { "post": { "operationId": "run-sync-data-done-right-remove-duplicates", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "items": { "title": "Items to deduplicate (pasted JSON)", "type": "array", "description": "Rows pasted as a JSON array of objects. Example: [{\"name\": \"Ann Lee\", \"email\": \"ann@x.com\"}, {\"name\": \"ann lee\", \"email\": \"ANN@x.com\"}]. Works offline, so it is the quickest way to try the Actor. You can combine it with the other sources below. Leave empty if you only use a dataset, file links or text lines." }, "datasetIds": { "title": "Apify datasets", "type": "array", "description": "Datasets to deduplicate. Pick them from the list, or paste dataset IDs. Reads go through your run, so datasets of other Actors and runs that you can read all work. Example: [\"aBcD1234eFgH5678i\", \"username~my-named-dataset\"]." }, "datasetUrls": { "title": "Dataset links or IDs", "type": "array", "description": "Dataset IDs, \"username~name\" names or Apify links, one per line. Example: https://console.apify.com/storage/datasets/aBcD1234eFgH5678i. A dataset that is only given here may be unreadable when the run has limited permissions: pick it in \"Apify datasets\" instead.", "items": { "type": "string" } }, "fileUrls": { "title": "File URLs (CSV, Excel, JSON, JSONL)", "type": "array", "description": "Direct links to .csv, .tsv, .xlsx, .json or .jsonl files, one per line. The type is detected from the extension or the content. Example: https://example.com/export/contacts.csv. Links to localhost and private networks are blocked.", "items": { "type": "string" } }, "textLines": { "title": "Plain text lines", "type": "string", "description": "One value per line, for example a pasted list of emails, URLs, names or phone numbers. Each non-empty line becomes a row with one field called text. Example: anna@acme.com (one per line). Keep the key fields empty to compare the whole line." }, "keyFields": { "title": "Key fields (what makes a duplicate)", "type": "array", "description": "Rows with the same value in all of these fields are duplicates. Example: email, or first name and last name on two lines. Dots reach nested fields: author.id. Leave empty to compare ALL fields of each row. A row where all key fields are empty is only removed if it is an exact copy of another row.", "items": { "type": "string" } }, "matchMode": { "title": "Match mode", "enum": [ "exact", "fuzzy" ], "type": "string", "description": "exact: rows are duplicates when their normalised keys are identical (fast, any size). fuzzy: exact copies first, then near matches above the similarity threshold (typos, word order, extra words). Fuzzy is slower; speed it up with blocking.", "default": "exact" }, "fuzzyAlgorithm": { "title": "Fuzzy algorithm", "enum": [ "tokenSet", "tokenSort", "jaroWinkler", "levenshtein" ], "type": "string", "description": "tokenSet: ignores word order, repeated words and extra words (best for names, companies, addresses). tokenSort: ignores word order only. jaroWinkler: typos in short strings with the same start. levenshtein: character edits. Only used in fuzzy mode.", "default": "tokenSet" }, "fuzzyThreshold": { "title": "Fuzzy threshold (similarity, 50-100)", "minimum": 50, "maximum": 100, "type": "integer", "description": "Two rows are near-duplicates when their similarity score is at least this value (100 = identical). 90 is a good start; lower finds more matches and more false ones. Example: 88. Only used in fuzzy mode.", "default": 90 }, "blocking": { "title": "Fuzzy blocking (speed)", "enum": [ "prefix", "phonetic", "tokens", "none" ], "type": "string", "description": "Rows are only compared with rows in the same block, which keeps 100,000+ rows fast. prefix: same first characters (misses pairs that differ in their first characters). phonetic: words that sound alike (Smith/Smyth). tokens: any shared word (most thorough, slower). none: compare everything (small data only).", "default": "prefix" }, "blockingChars": { "title": "Blocking characters", "minimum": 1, "maximum": 10, "type": "integer", "description": "For prefix blocking: how many leading characters of the key must match for two rows to be compared. 1-2 finds more pairs, 3-4 is faster on big data. Example: 2.", "default": 2 }, "normalise": { "title": "Clean text before comparing", "type": "array", "description": "Applied to every key value before comparing; your original data is never changed. trim removes spaces at the ends; caseFold makes upper and lower case equal (also ss for the German sharp s); collapseSpaces turns runs of spaces into one; stripPunctuation ignores dots, commas, dashes; stripAccents makes e and the accented letter equal. Default: trim, caseFold, collapseSpaces.", "items": { "type": "string", "enum": [ "trim", "caseFold", "collapseSpaces", "stripPunctuation", "stripAccents" ], "enumTitles": [ "Trim spaces", "Ignore upper/lower case", "Collapse repeated spaces", "Ignore punctuation", "Ignore accents (Jose = Jose)" ] }, "default": [ "trim", "caseFold", "collapseSpaces" ] }, "fieldTypes": { "title": "Smart normaliser per field", "type": "object", "description": "Pick a normaliser for fields that hold emails, URLs, phones or domains, as {\"field\": \"type\"}. Types: email (case, Name forms), url (no http/https, www, trailing slash, utm_ and click-id parameters), phone (digits only), domain (host of a URL or email), text. Example: {\"email\": \"email\", \"website\": \"url\", \"phone\": \"phone\"}." }, "emailGmailDots": { "title": "Gmail: ignore dots in the name part", "type": "boolean", "description": "Treat first.last@gmail.com and firstlast@gmail.com as the same address (Gmail ignores dots). Only for gmail.com and googlemail.com. Applies to fields with the email normaliser.", "default": false }, "emailPlusTags": { "title": "Ignore +tags in emails", "type": "boolean", "description": "Treat anna+news@acme.com and anna@acme.com as the same address. Many providers (Gmail, Outlook, Fastmail) deliver both to the same inbox. Applies to fields with the email normaliser.", "default": false }, "phoneLastDigits": { "title": "Phone: compare only the last N digits", "minimum": 0, "maximum": 15, "type": "integer", "description": "0 compares all digits. Set 9 or 10 to ignore country codes and trunk prefixes, so +44 20 7946 0958 and 020 7946 0958 match on the last 9 digits. Example: 9. Applies to fields with the phone normaliser.", "default": 0 }, "keepStrategy": { "title": "Which row to keep", "enum": [ "first", "last", "mostComplete", "longestField" ], "type": "string", "description": "first: keep the earliest row (lowest memory use, streams). last: keep the latest. mostComplete: keep the row with the most non-empty fields. longestField: keep the row whose Keep-field value is longest. Ties go to the earliest row. Kept rows stay at the position of the group's first row.", "default": "first" }, "keepField": { "title": "Field for 'longest value'", "type": "string", "description": "Only for the longestField strategy: the field whose longest text wins, e.g. description. Dots reach nested fields." }, "mergeDuplicates": { "title": "Merge duplicates into one enriched row", "type": "boolean", "description": "Fill empty fields of the kept row with values from its duplicates (a phone number found in a second copy, for example). Never overwrites a value that is already there. Holds the kept rows in memory.", "default": false }, "outputMode": { "title": "What to put in the dataset", "enum": [ "unique", "duplicates" ], "type": "string", "description": "unique: the clean rows without duplicates. duplicates: only the rows that were removed, each with clusterId, duplicateOf (the row number that was kept), matchReason and matchScore, to inspect what was found.", "default": "unique" }, "addDuplicateCount": { "title": "Add duplicateCount column", "type": "boolean", "description": "Add a duplicateCount column: how many duplicates were folded into each unique row (0 = it had none). If your data already has a column of that name, the new one is called _duplicateCount.", "default": false }, "addClusterId": { "title": "Add clusterId column", "type": "boolean", "description": "Add a clusterId column (1, 2, 3...) that identifies each group of duplicates; the same ID appears in the duplicates report. If your data already has that column, the new one is called _clusterId.", "default": false }, "duplicatesReport": { "title": "Write the duplicates report", "type": "boolean", "description": "Save duplicates-report.json and duplicates-report.csv in the key-value store: every dropped row with its source, row number, why it matched (exact or fuzzy with score), the key it matched, and the row number that was kept. Free of charge.", "default": true }, "maxReportRows": { "title": "Max rows in the report", "minimum": 0, "maximum": 200000, "type": "integer", "description": "Longest the duplicates report can get. The summary counts always cover all rows. Example: 10000.", "default": 10000 }, "exports": { "title": "Download files of the clean data", "type": "array", "description": "Also write the clean data as files in the key-value store, each with a download link in the SUMMARY record. Each file is charged once (export-file). Leave empty for the dataset only.", "items": { "type": "string", "enum": [ "csv", "xlsx", "json", "jsonl" ], "enumTitles": [ "CSV (UTF-8, opens in Excel)", "Excel (.xlsx)", "JSON array", "JSON Lines" ] }, "default": [] }, "exportName": { "title": "Export file name", "type": "string", "description": "Base name of the files, e.g. clean-data gives clean-data.csv and clean-data.xlsx.", "default": "clean-data" }, "escapeFormulas": { "title": "Protect CSV from spreadsheet formulas", "type": "boolean", "description": "In the CSV export and report, text that starts with = + - or @ gets a leading apostrophe so Excel and Sheets cannot run it as a formula. Excel files always store such text as plain text. Turn off only if you need the raw values.", "default": true }, "maxItems": { "title": "Max rows to process", "minimum": 1, "maximum": 5000000, "type": "integer", "description": "Hard cap on input rows across all sources. Reading stops at the cap and the run still succeeds. You are charged per row processed, so this also caps cost. Example: 100000.", "default": 500000 }, "maxItemsPerSource": { "title": "Max rows per source", "minimum": 1, "maximum": 5000000, "type": "integer", "description": "Optional cap on rows taken from each source, so one huge source cannot use up the whole budget. Leave empty for no per-source cap." }, "maxFileSizeMb": { "title": "Max file size (MB)", "minimum": 1, "maximum": 500, "type": "integer", "description": "Files larger than this are rejected (that source fails, the others continue). A JSON array file is loaded whole and is limited to 100 MB; use JSONL or CSV for bigger files.", "default": 50 }, "csvInferTypes": { "title": "Detect numbers and booleans in CSV", "type": "boolean", "description": "Turn CSV text like 42, 3.5 or true into numbers and booleans. Values with leading zeros (007), plus signs or very long digit strings stay text. Matching treats 42 and \"42\" as equal either way.", "default": true }, "xlsxSheet": { "title": "Excel sheet", "type": "string", "description": "Sheet name or number (1 = first) to read from every Excel file. Leave empty for the first sheet." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }