{ "openapi": "3.0.1", "info": { "title": "Merge & Deduplicate Datasets", "description": "Merge many Apify datasets into one and remove duplicates by any fields, including nested dot paths. Keep the first or last copy, pick and rename fields. Streams page by page so memory stays low on large datasets. Limited permissions, clear errors before any work.", "version": "0.1", "x-build-id": "1jW2h1pB0rmu1M2T2" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/ctriolab~merge-dedupe-datasets/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-ctriolab-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/ctriolab~merge-dedupe-datasets/runs": { "post": { "operationId": "runs-sync-ctriolab-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/ctriolab~merge-dedupe-datasets/run-sync": { "post": { "operationId": "run-sync-ctriolab-merge-dedupe-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "datasetIds" ], "properties": { "datasetIds": { "title": "Datasets to merge", "minItems": 1, "type": "array", "description": "Pick one or more datasets (or paste dataset IDs). Items are read in this order. Tip: a run's results are its default dataset, shown on the run's Storage tab." }, "dedupeFields": { "title": "Deduplicate by fields", "type": "array", "description": "Items with the same combination of these values are duplicates. Use dots for nested values, e.g. \"address.city\" or \"images.0.url\". Leave empty to only merge.", "items": { "type": "string" } }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last" ], "type": "string", "description": "\"First\" keeps the earliest item (dataset order, then item order). \"Last\" keeps the latest one, for example the newest scrape when datasets are listed oldest first. \"Last\" reads the input twice.", "default": "first" }, "normalizeValues": { "title": "Ignore case and extra spaces", "type": "boolean", "description": "Compare values after trimming, collapsing spaces and lowercasing, so \"ACME Inc\" equals \"acme inc\" and 1 equals \"1\". Turn off for exact comparison. Objects and arrays are compared by content.", "default": true }, "fields": { "title": "Output only these fields", "type": "array", "description": "Optional. Keep only these fields (dot paths allowed) in the output, in this order. Missing values become null. Leave empty to keep whole items.", "items": { "type": "string" } }, "rename": { "title": "Rename fields", "type": "object", "description": "Optional. Map a field or dot path to a new output name, e.g. {\"address.city\": \"city\", \"title\": \"name\"}." }, "pageSize": { "title": "Items per page", "minimum": 100, "maximum": 10000, "type": "integer", "description": "How many items are read per API call. Lower it for very large items (for example full HTML) to keep memory low.", "default": 1000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }