{ "openapi": "3.0.1", "info": { "title": "Remove Duplicates: Deduplicate & Merge Datasets by Field", "description": "Remove duplicates: merge several Apify datasets into one and remove duplicate rows by the fields you choose (e.g. url, placeId, email). Case, whitespace and URL normalization, keep first or last, pick or omit fields. Start with no dataset for a free demo: 5 sample rows in, 3 unique rows out.", "version": "0.1", "x-build-id": "ULmoGcLQsGQncNzJh" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/ucchiii~dataset-dedup-merge/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-ucchiii-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/ucchiii~dataset-dedup-merge/runs": { "post": { "operationId": "runs-sync-ucchiii-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/ucchiii~dataset-dedup-merge/run-sync": { "post": { "operationId": "run-sync-ucchiii-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Datasets", "type": "array", "description": "Dataset IDs (the 'defaultDatasetId' of a finished run of the source Actor, e.g. 'WkzbQMuFYuamGv3YF'). Dataset IDs or names to merge, in order. Items are read in this order (this matters for 'keep first/last'). Leave empty to run a free demo on sample data (not charged). You can also paste the dataset URL from Apify Console or the API (the ID is taken from it)." }, "dedupFields": { "title": "Deduplicate by fields", "type": "array", "description": "Items with the same values in ALL these fields are duplicates. Use dots for nested fields (e.g. ['url'] or ['address.city']). Empty = compare whole items.", "items": { "type": "string" } }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last" ], "type": "string", "description": "When duplicates are found, keep the 'first' or the 'last' occurrence (datasets are read in the order given). Default: first.", "default": "first" }, "output": { "title": "Output", "enum": [ "unique", "duplicatesOnly" ], "type": "string", "description": "'unique' returns the deduplicated items (default). 'duplicatesOnly' returns only the duplicates that were removed (for auditing).", "default": "unique" }, "caseInsensitive": { "title": "Ignore letter case", "type": "boolean", "description": "Treat 'Acme' and 'acme' as the same value.", "default": true }, "trimWhitespace": { "title": "Ignore extra whitespace", "type": "boolean", "description": "Treat ' Acme Inc ' and 'Acme Inc' as the same value.", "default": true }, "normalizeUrls": { "title": "Normalize URLs", "type": "boolean", "description": "Treat http/https, www and a trailing slash as the same URL.", "default": false }, "dedupEmptyKeys": { "title": "Deduplicate items with empty key fields", "type": "boolean", "description": "By default, items whose dedup fields are all empty are kept (never treated as duplicates).", "default": false }, "fields": { "title": "Keep only these fields", "type": "array", "description": "Top-level fields to output. Empty = all fields.", "items": { "type": "string" } }, "omitFields": { "title": "Remove these fields", "type": "array", "description": "Top-level fields to remove from the output.", "items": { "type": "string" } }, "addSourceField": { "title": "Add source dataset ID", "type": "boolean", "description": "Adds '_sourceDatasetId' to each output item.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }