{ "openapi": "3.0.1", "info": { "title": "Merge & Dedup Datasets: Remove Duplicates, Only New Items", "description": "Merge any number of Apify datasets and remove duplicates by the fields you choose (or the whole item). Keep the first or newest copy, or output only items not seen in earlier runs: ideal after a scheduled scraper. Streams millions of items. Duplicates are free; $0.0019 per 1,000 items output.", "version": "0.0", "x-build-id": "pmxpm2SSDrBdZMTr9" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/charlesmarkette~merge-deduplicate-datasets-only-new-items/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-charlesmarkette-merge-deduplicate-datasets-only-new-items", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/charlesmarkette~merge-deduplicate-datasets-only-new-items/runs": { "post": { "operationId": "runs-sync-charlesmarkette-merge-deduplicate-datasets-only-new-items", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/charlesmarkette~merge-deduplicate-datasets-only-new-items/run-sync": { "post": { "operationId": "run-sync-charlesmarkette-merge-deduplicate-datasets-only-new-items", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Datasets", "type": "array", "description": "One or more datasets to merge, in order. In an integration, use [\"{{resource.defaultDatasetId}}\"]. Leave empty to use 'Items' below instead." }, "items": { "title": "Items (instead of datasets)", "type": "array", "description": "Optional. A JSON array of objects to deduplicate, used when no dataset is picked. Handy for quick tests and for calling this Actor from code." }, "mode": { "title": "Mode", "enum": [ "dedupe", "onlyNew", "merge" ], "type": "string", "description": "'Only new' remembers what earlier runs of this saved task (or this exact setup) returned and outputs just the new items.", "default": "dedupe" }, "dedupFields": { "title": "Deduplicate by fields", "type": "array", "description": "Items with the same values in these fields are duplicates, e.g. 'url' or 'id'. Use dots for nested fields ('seller.id'). Empty = compare whole items.", "items": { "type": "string" } }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last" ], "type": "string", "description": "With several datasets, 'last' keeps the version from the later dataset.", "default": "first" }, "caseInsensitive": { "title": "Ignore letter case in key fields", "type": "boolean", "description": "'ACME' and 'acme' count as the same.", "default": false }, "trimWhitespace": { "title": "Ignore extra spaces in key fields", "type": "boolean", "description": "Trims and collapses whitespace before comparing.", "default": false }, "requiredFields": { "title": "Drop items missing these fields", "type": "array", "description": "Items where any of these fields is missing or empty are skipped.", "items": { "type": "string" } }, "outputFields": { "title": "Output only these fields", "type": "array", "description": "Optional field list (dots for nested). Empty = keep all fields.", "items": { "type": "string" } }, "omitFields": { "title": "Remove these fields", "type": "array", "description": "Top-level fields to drop from the output.", "items": { "type": "string" } }, "addSourceDataset": { "title": "Add _sourceDataset field", "type": "boolean", "description": "Adds the ID of the dataset each item came from.", "default": false }, "maxItems": { "title": "Max output items", "minimum": 1, "type": "integer", "description": "Optional cap on items written." }, "stateName": { "title": "Monitoring name (Only new mode)", "type": "string", "description": "Optional. Runs with the same name share their memory of seen items, even when the input dataset changes every run (typical for scheduled scrapers). Leave empty to tie memory to the saved task or to these exact datasets." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }