{ "openapi": "3.0.1", "info": { "title": "Remove Duplicates & Merge CSV, JSON and Datasets", "description": "Merge Apify datasets, CSV and JSON files into one table and remove duplicates by any fields (url, email, website ...), keeping the first or newest row. Loose matching of case, spaces and URLs. $0.10 per 1,000 rows out.", "version": "0.1", "x-build-id": "0NP8k8dF3ZF0GoONc" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rowfeed~dataset-merge-dedup/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rowfeed-dataset-merge-dedup", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rowfeed~dataset-merge-dedup/runs": { "post": { "operationId": "runs-sync-rowfeed-dataset-merge-dedup", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rowfeed~dataset-merge-dedup/run-sync": { "post": { "operationId": "run-sync-rowfeed-dataset-merge-dedup", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Apify datasets", "type": "array", "description": "Datasets to merge, e.g. the results of several runs of any Actor. Pick them or paste dataset IDs (Storage tab, or the run's Output)." }, "fileUrls": { "title": "File URLs (CSV, JSON, JSON Lines)", "type": "array", "description": "Public links to CSV (comma, semicolon or tab), JSON array or JSON Lines files, up to 50 MB each. Excel: save as CSV first.", "items": { "type": "string" } }, "rows": { "title": "Rows (pasted JSON)", "type": "array", "description": "Or paste rows as a JSON array of objects." }, "dedupFields": { "title": "Remove duplicates by these fields", "type": "array", "description": "Rows with the same values in these fields count as duplicates, e.g. url, email, website or title + city. Dots reach nested fields (address.city). Empty = only rows that are identical in every field.", "items": { "type": "string" } }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last" ], "type": "string", "description": "\"Last\" keeps the newest version of each row, e.g. when merging today's run into yesterday's.", "default": "first" }, "looseMatch": { "title": "Loose matching", "type": "boolean", "description": "Ignore upper/lower case and extra spaces, and treat http/https, www. and a trailing slash in URLs as the same.", "default": true }, "outputFields": { "title": "Output only these fields", "type": "array", "description": "Optional: keep only these columns, in this order (dots flatten nested fields). Empty = all fields.", "items": { "type": "string" } }, "addSource": { "title": "Add source column", "type": "boolean", "description": "Add _source to each row: which dataset, file or input the row came from.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }