{ "openapi": "3.0.1", "info": { "title": "Dataset Dedup and Merge API: Remove Duplicate Rows", "description": "Merge several runs into one clean dataset and pay only for the unique rows delivered. Choose the key columns, normalise case, spaces and URLs, keep the most complete row and get a duplicate report. Works inside your own Apify account.", "version": "1.0", "x-build-id": "pQaZsYrNAQnOLSDIE" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rowcraft~dataset-dedup-merge/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rowcraft-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rowcraft~dataset-dedup-merge/runs": { "post": { "operationId": "runs-sync-rowcraft-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rowcraft~dataset-dedup-merge/run-sync": { "post": { "operationId": "run-sync-rowcraft-dataset-dedup-merge", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "datasetIds" ], "properties": { "datasetIds": { "title": "Datasets to merge", "minItems": 1, "maxItems": 20, "type": "array", "description": "The datasets to read, from your own Apify account: 1 to 20. Paste dataset IDs (from the Storage tab, or the Dataset ID of a run) or `username~dataset-name` for a named dataset. Rows are read in the order given. Rows of a dataset that is not found are skipped and reported in COVERAGE. The prefilled IDs are two public sample datasets of the Rowcraft account (two runs of the Hacker News front page), so you can press Start and see a merge right away.", "items": { "type": "string" } }, "keyFields": { "title": "Key fields (what makes a row a duplicate)", "type": "array", "description": "Columns whose values decide whether two rows are the same, one per line, for example `url` or `productUrl`; a nested column is written with dots, for example `seller.id`. Two rows are duplicates when all of these values are equal after normalising. Leave empty to compare the whole row.", "default": [], "items": { "type": "string" } }, "normalize": { "title": "Normalise values before comparing", "type": "array", "description": "How values are cleaned before they are compared (only for the comparison; the delivered rows keep their original values). Default: lower case, trim, collapse spaces. Clear all to compare exactly. Query and trailing-slash options apply to values that are web addresses; digits only keeps just the digits (phone-like codes, IDs).", "items": { "type": "string", "enum": [ "lowercase", "trim", "collapseWhitespace", "stripUrlQuery", "stripUrlTrailingSlash", "digitsOnly" ], "enumTitles": [ "Lower case", "Trim spaces at both ends", "Collapse repeated spaces", "Web addresses: drop ?query and #fragment", "Web addresses: drop trailing slash", "Keep digits only" ] }, "default": [ "lowercase", "trim", "collapseWhitespace" ] }, "keep": { "title": "Which copy to keep", "enum": [ "first", "last", "mostComplete" ], "type": "string", "description": "When several rows have the same key: the first one in the order of your datasets (default), the last one (the newest run, if you list the runs oldest first), or the row with the most filled-in fields.", "default": "first" }, "mergeFields": { "title": "Fill empty fields from the duplicates", "type": "boolean", "description": "After choosing the row to keep, fill its empty fields (null, empty text, empty list) with the first non-empty value found in its duplicates. Top-level fields only. Reads your datasets one more time.", "default": false }, "addSourceColumns": { "title": "Add source columns", "type": "boolean", "description": "Add `_sourceDatasetId` (the dataset the row comes from), `_sourceIndex` (its position in that dataset, starting at 0) and `_duplicateCount` (how many copies were removed) to every delivered row.", "default": true }, "fieldsToKeep": { "title": "Columns to keep", "type": "array", "description": "Deliver only these top-level columns, in this order, one per line. Leave empty to keep all. With no key fields the comparison uses the row as delivered, so a column left out here is ignored when comparing.", "default": [], "items": { "type": "string" } }, "fieldsToDrop": { "title": "Columns to drop", "type": "array", "description": "Remove these top-level columns from the delivered rows, one per line, for example a scrape time that differs between runs. With no key fields a dropped column is also ignored when comparing.", "default": [], "items": { "type": "string" } }, "maxItems": { "title": "Maximum rows delivered", "minimum": 0, "type": "integer", "description": "Stop after this many unique rows are delivered. 0 = no limit. A run reads at most 500,000 rows in all, from at most 20 datasets.", "default": 0 }, "dryRun": { "title": "Dry run (statistics only)", "type": "boolean", "description": "Read the datasets and write only the statistics to the DEDUP_REPORT and RUN_SUMMARY records: rows read, unique rows, duplicates per dataset and the estimated cost. Nothing is delivered and nothing is charged apart from the start event.", "default": false }, "onlyNew": { "title": "Only new rows since the last run", "type": "boolean", "description": "Remember the rows already delivered to you (in a key-value store in your Apify account) and deliver only rows that appeared since the last run with the same state key. The first run delivers everything that matches; later runs cost only the new rows. Ideal for schedules.", "default": false }, "stateName": { "title": "State name", "type": "string", "description": "Name of the memory used by \"Only new rows\". Use a different name for each schedule or search (for example `stockholm-python`) so they do not share one memory.", "default": "default" }, "notifyUrl": { "title": "Notify this URL when the run ends", "type": "string", "description": "A webhook address that receives one JSON POST with the number of new rows and links to the dataset: a Slack or Discord incoming webhook, a Make, n8n or Zapier catch hook, or your own endpoint. The message is also in the `text` (Slack) and `content` (Discord) fields." }, "notifyWhenEmpty": { "title": "Notify also when there are no new rows", "type": "boolean", "description": "By default the notification is sent only when at least one row was delivered.", "default": false }, "requestIntervalMs": { "title": "Pause between reads from the Apify API (ms)", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Pause after each page of rows read from your datasets. Default 0: the API of your own account needs no politeness pause. Raise it only if a run hits rate limits.", "default": 0 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }