{ "openapi": "3.0.1", "info": { "title": "Merge Datasets & Remove Duplicates — Deduplicate, Filter, Excel", "description": "Merge datasets and remove duplicates in one call: combine Apify datasets, CSV, Excel, JSON or Google Sheets, dedupe by any field (smart URL/email/phone matching), filter, sort, keep only new rows, export CSV/Excel/JSON. Works from AI agents via Apify MCP (Claude, ChatGPT), n8n and the API.", "version": "1.0", "x-build-id": "GG0WWCmB0xNSuzy5v" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/tidydata~merge-dedupe-datasets/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-tidydata-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/tidydata~merge-dedupe-datasets/runs": { "post": { "operationId": "runs-sync-tidydata-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/tidydata~merge-dedupe-datasets/run-sync": { "post": { "operationId": "run-sync-tidydata-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Datasets to merge", "type": "array", "description": "Pick one or more Apify datasets from your storages (or paste their IDs). They are merged in this order. When this Actor runs as an integration, use {{resource.defaultDatasetId}}." }, "fileUrls": { "title": "Or files to merge (CSV, Excel, JSON, Google Sheets)", "type": "array", "description": "Public links to .csv, .xlsx or .json files (JSON: a list of objects or JSON Lines), or Google Sheets shared as \"Anyone with the link can view\". Added after the datasets.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "dedupeFields": { "title": "Remove duplicates by these fields", "type": "array", "description": "Rows with the same values in ALL these fields count as duplicates (e.g. url, or id). Nested fields: price.value. Leave empty to keep every row.", "items": { "type": "string" } }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last", "merge" ], "type": "string", "description": "When several rows are duplicates, keep the first one found or the last one (most recent). \"Merge them\" keeps one row per duplicate group and fills its empty fields with values from the other copies (adds _duplicates = how many were merged).", "default": "first" }, "normalizeKeys": { "title": "Smart duplicate matching", "type": "boolean", "description": "Ignore upper/lower case and extra spaces when comparing duplicates; URLs also ignore http/https, www., trailing slash, #anchors and tracking parameters (utm_*, fbclid, gclid…); phone numbers compare digits only. Note: switching it changes what 'Only new rows' remembers.", "default": false }, "keepRowsWithoutKey": { "title": "Keep rows with empty duplicate fields", "type": "boolean", "description": "By default all rows where the duplicate fields are empty count as one duplicate group. Turn on to keep every such row (e.g. dedupe by email but keep the leads that have no email).", "default": false }, "onlyNew": { "title": "Only rows not seen in earlier runs", "type": "boolean", "description": "Remember the duplicate keys of every run (needs 'Remove duplicates by' fields) and output only rows that are new. Great for scheduled scrapers.", "default": false }, "onlyNewMemoryName": { "title": "Memory name (for 'only new')", "type": "string", "description": "Runs that share this name share their memory. Use a different name per project.", "default": "merge-dedupe-memory" }, "filters": { "title": "Keep only rows where…", "type": "array", "description": "One rule per line: field operator value. Operators: =, !=, >, >=, <, <=, contains, not_contains, starts_with, ends_with, in (comma list), not_in, regex, not_regex, exists, not_exists. Examples: price < 50 · country in ES, FR, IT · title contains phone · email exists · url regex /product/", "items": { "type": "string" } }, "filterMode": { "title": "Rows must match", "enum": [ "all", "any" ], "type": "string", "description": "Whether a row needs to pass every filter or just one of them.", "default": "all" }, "exportFormats": { "title": "Also create files", "type": "array", "description": "Download links appear in the Output tab. Each file created costs a small fixed fee.", "items": { "type": "string", "enum": [ "csv", "xlsx", "json" ] }, "default": [ "csv" ] }, "computedFields": { "title": "Add computed columns", "type": "array", "description": "One per line: newColumn = template. Use {field} and functions with |: {url|domain}, {name|lower}, {price|number}, {phone|digits}, {title|trim}, {tags|first}, {tags|length}, {text|nohtml}, {name|slug}, {name|upper}, {name|title}. Example: fullName = {firstName} {lastName}. Computed columns can be used in filters and duplicates.", "items": { "type": "string" } }, "unwindField": { "title": "Split a list column into rows", "type": "string", "description": "Name of a field that holds a list (e.g. emails, reviews, variants): you get one row per item, the other columns are repeated. Done before flattening." }, "sortBy": { "title": "Sort by", "type": "array", "description": "One per line: field asc|desc, e.g. rating desc, then price asc. Numbers like '12,50 €' sort as numbers; empty values go last.", "items": { "type": "string" } }, "selectFields": { "title": "Keep only these columns", "type": "array", "description": "Original column names, in the order you want them. Leave empty to keep all.", "items": { "type": "string" } }, "dropFields": { "title": "Remove these columns", "type": "array", "description": "Columns to remove from the result. One name per line (nested: price.value).", "items": { "type": "string" } }, "renameFields": { "title": "Rename columns", "type": "object", "description": "Old name → new name. Example: {\"price.value\": \"price\"}" }, "addSourceField": { "title": "Add a _source column", "type": "boolean", "description": "Adds _source with the dataset ID or file link each row came from.", "default": false }, "flatten": { "title": "Flatten nested data", "type": "boolean", "description": "Turn {\"price\": {\"value\": 5}} into a column price.value. Needed for CSV/Excel.", "default": true }, "maxRows": { "title": "Max rows in the result", "minimum": 1, "maximum": 2000000, "type": "integer", "description": "Stop after this many rows in the result.", "default": 500000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }