{
  "openapi": "3.0.1",
  "info": {
    "title": "Merge & Deduplicate Datasets: Only New Items, CSV/Excel Export",
    "description": "Merge and deduplicate Apify datasets and CSV, JSON, Excel or Parquet files in one run. Remove duplicates (exact, normalized or fuzzy), output only new items since your last run, join / VLOOKUP a second list by key, filter, sort, then export to CSV, Excel, JSON or Parquet. $0.002 per 1,000 rows.",
    "version": "1.0",
    "x-build-id": "ULEbsJvs1Xsodl3CW"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/tinplatelabs~dataset-toolkit/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-tinplatelabs-dataset-toolkit",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/tinplatelabs~dataset-toolkit/runs": {
      "post": {
        "operationId": "runs-sync-tinplatelabs-dataset-toolkit",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/tinplatelabs~dataset-toolkit/run-sync": {
      "post": {
        "operationId": "run-sync-tinplatelabs-dataset-toolkit",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "datasetIds": {
            "title": "Datasets to merge and deduplicate",
            "type": "array",
            "description": "Apify datasets to merge, e.g. the outputs of several scraper runs: pick them or paste dataset IDs (or <code>username~dataset-name</code>). Combine with file URLs and inline items; everything becomes one table that is deduplicated, filtered and exported. Another user's dataset must be shared with you or public. Read page by page, so size is not limited by memory. Turn on <b>Only new rows</b> below to output only rows not delivered by earlier runs (for scheduled scrapers)."
          },
          "fileUrls": {
            "title": "File URLs",
            "type": "array",
            "description": "Public URLs of CSV, TSV, JSON (array), JSON Lines, XLSX (first sheet) or Parquet files, optionally gzip-compressed. Format is detected from the extension, Content-Type or content. Up to 1 GB each.",
            "items": {
              "type": "string"
            }
          },
          "items": {
            "title": "Inline items",
            "type": "array",
            "description": "Rows given directly as a JSON array of objects, e.g. from an AI agent or an API call. Read before datasets and files."
          },
          "fileFormat": {
            "title": "File format",
            "enum": [
              "auto",
              "csv",
              "tsv",
              "json",
              "jsonl",
              "xlsx",
              "parquet"
            ],
            "type": "string",
            "description": "Force the format of every file URL. Auto detects it per file.",
            "default": "auto"
          },
          "csvDelimiter": {
            "title": "CSV delimiter",
            "enum": [
              "auto",
              ",",
              ";",
              "\\t",
              "|"
            ],
            "type": "string",
            "description": "Delimiter of CSV files. Auto picks the most frequent of , ; tab | in the header line.",
            "default": "auto"
          },
          "sourceField": {
            "title": "Source column",
            "type": "string",
            "description": "Name of a column recording where each row came from (<code>inline</code>, <code>dataset:&lt;id&gt;</code> or the file URL), e.g. <code>_source</code>. Empty = no column. Ignored by whole-row dedupe."
          },
          "joinDatasetId": {
            "title": "Join / VLOOKUP: lookup dataset",
            "type": "string",
            "description": "Enrich your rows with a second source, like VLOOKUP or a SQL join: pick the Apify dataset to look values up in (or paste its ID or <code>username~dataset-name</code>). Give one lookup source: this dataset, a lookup file URL or lookup items. Set the key in <b>Join on</b>."
          },
          "joinFileUrl": {
            "title": "Join / VLOOKUP: lookup file URL",
            "type": "string",
            "description": "Or a public CSV, TSV, JSON, JSON Lines, XLSX or Parquet file to join with (format detected), e.g. a CRM export or a price list."
          },
          "joinItems": {
            "title": "Join / VLOOKUP: lookup items",
            "type": "array",
            "description": "Or the lookup rows inline, as a JSON array of objects."
          },
          "joinOn": {
            "title": "Join on (key fields)",
            "type": "array",
            "description": "Field(s) that must match, e.g. <code>email</code>. If the lookup names it differently: <code>email=Email Address</code> (main field = lookup field). Several fields are combined into one key.",
            "items": {
              "type": "string"
            }
          },
          "joinType": {
            "title": "Join type",
            "enum": [
              "left",
              "inner",
              "anti"
            ],
            "type": "string",
            "description": "<b>left</b>: keep every row and add the lookup's columns where a match is found (VLOOKUP). <b>inner</b>: keep only rows with a match. <b>anti</b>: keep only rows with no match, i.e. rows not in the other list.",
            "default": "left"
          },
          "joinFields": {
            "title": "Join: columns to bring in",
            "type": "array",
            "description": "Lookup columns to add to matched rows. Empty: all of them except the key.",
            "items": {
              "type": "string"
            }
          },
          "joinMatch": {
            "title": "Join: key matching",
            "enum": [
              "normalized",
              "exact",
              "fuzzy"
            ],
            "type": "string",
            "description": "<b>normalized</b> ignores case, extra spaces and Unicode width (<code> Ann@X.com</code> = <code>ann@x.com</code>). <b>fuzzy</b> also tolerates typos (uses <b>Fuzzy threshold</b>). The first lookup row per key is used.",
            "default": "normalized"
          },
          "joinPrefix": {
            "title": "Join: prefix for clashing columns",
            "type": "string",
            "description": "A lookup column whose name the row already has is added as prefix + name, e.g. <code>joined_name</code>.",
            "default": "joined_"
          },
          "flatten": {
            "title": "Flatten nested fields",
            "enum": [
              "none",
              "objects",
              "all"
            ],
            "type": "string",
            "description": "Turn nested JSON into columns before filtering: <b>objects</b> makes <code>address.city</code> columns; <b>all</b> also splits arrays (<code>tags.0</code>). Without flattening, nested values become JSON text in CSV/XLSX.",
            "default": "none"
          },
          "flattenSeparator": {
            "title": "Flatten separator",
            "type": "string",
            "description": "Joins nested names when flattening, e.g. <code>.</code> or <code>/</code>.",
            "default": "."
          },
          "filter": {
            "title": "Filter",
            "type": "string",
            "description": "Keep only rows matching this rule, e.g. <code>price &gt;= 10 and country in [\"CA\", \"US\"] and not email empty</code>. Operators: = != &lt; &lt;= &gt; &gt;= contains startswith endswith matches (regex) in exists empty, combined with and / or / not and parentheses. Nested fields: <code>address.city</code>; names with spaces in `backticks`. See the README."
          },
          "dedupe": {
            "title": "Remove duplicates",
            "enum": [
              "none",
              "exact",
              "normalized",
              "fuzzy"
            ],
            "type": "string",
            "description": "<b>exact</b>: identical values. <b>normalized</b>: ignores case, extra spaces and Unicode width. <b>fuzzy</b>: near matches such as typos (rapidfuzz similarity at least the threshold).",
            "default": "none"
          },
          "dedupeFields": {
            "title": "Dedupe on fields",
            "type": "array",
            "description": "Fields that identify a duplicate, e.g. <code>email</code> or <code>name</code> + <code>city</code>. Empty = the whole row. Rows with all these fields empty are always kept.",
            "items": {
              "type": "string"
            }
          },
          "fuzzyThreshold": {
            "title": "Fuzzy threshold",
            "minimum": 50,
            "maximum": 100,
            "type": "integer",
            "description": "Fuzzy mode: similarity (0-100, token sort ratio) needed to count as a duplicate. 90 catches typos and word order; lower values merge more aggressively.",
            "default": 90
          },
          "keep": {
            "title": "Which duplicate to keep",
            "enum": [
              "first",
              "last",
              "most-complete"
            ],
            "type": "string",
            "description": "Keep the first row seen, the last one, or the one with the most non-empty fields.",
            "default": "first"
          },
          "duplicateCountField": {
            "title": "Duplicate count column",
            "type": "string",
            "description": "Optional column name that receives how many rows were in each kept row's duplicate group, e.g. <code>duplicates</code>."
          },
          "onlyNew": {
            "title": "Only new rows (skip rows from earlier runs)",
            "type": "boolean",
            "description": "Output only rows that earlier runs with the same <b>memory name</b> did not deliver: ideal after a scheduled scraper, so each run passes on just the new items. Rows are matched on <code>dedupeFields</code> (or the whole row), exactly, or ignoring case and spaces when dedupe is normalized or fuzzy. The memory is a key-value store <code>toolkit-only-new-&lt;name&gt;</code> in your account (up to 5 million rows; delete it to start over). Rows removed by the filter or the limit are not remembered. No extra charge: rows read are priced as usual.",
            "default": false
          },
          "onlyNewKey": {
            "title": "Only new rows: memory name",
            "type": "string",
            "description": "Name of the memory, e.g. <code>daily-leads</code>. Use one name per pipeline; a new name starts with an empty memory.",
            "default": "default"
          },
          "sortBy": {
            "title": "Sort by",
            "type": "array",
            "description": "Fields to sort by, in order. Prefix with <code>-</code> or add <code>:desc</code> for descending; add <code>:number</code> or <code>:text</code> to force a type. Numbers sort numerically (also in text), text ignores case, missing values go last. You can also sort by the duplicate count column.",
            "items": {
              "type": "string"
            }
          },
          "limit": {
            "title": "Max rows",
            "minimum": 0,
            "type": "integer",
            "description": "Keep at most this many rows after filtering, dedupe and sorting. 0 = all.",
            "default": 0
          },
          "fields": {
            "title": "Keep only these fields",
            "type": "array",
            "description": "Output exactly these columns, in this order (nested paths like <code>address.city</code> allowed). Empty = all fields.",
            "items": {
              "type": "string"
            }
          },
          "omitFields": {
            "title": "Remove fields",
            "type": "array",
            "description": "Columns to drop (when 'Keep only these fields' is empty).",
            "items": {
              "type": "string"
            }
          },
          "rename": {
            "title": "Rename fields",
            "type": "object",
            "description": "Old name to new name, e.g. <code>{\"Email Address\": \"email\"}</code>. Applied last."
          },
          "exportFormats": {
            "title": "Export files",
            "type": "array",
            "description": "Files written to the run's key-value store, with download links in the SUMMARY record. CSV, XLSX and Parquet use every column seen; nested values become JSON text.",
            "items": {
              "type": "string",
              "enum": [
                "csv",
                "xlsx",
                "json",
                "jsonl",
                "parquet"
              ],
              "enumTitles": [
                "CSV",
                "Excel (XLSX)",
                "JSON",
                "JSON Lines",
                "Parquet"
              ]
            }
          },
          "exportFileName": {
            "title": "Export file name",
            "type": "string",
            "description": "Base name of the export files (extension added).",
            "default": "output"
          },
          "datasetOutput": {
            "title": "Dataset output",
            "enum": [
              "rows",
              "summary",
              "none"
            ],
            "type": "string",
            "description": "<b>rows</b>: the processed rows go to the run's dataset. <b>summary</b>: one item with counts and export links (handy for agents and huge files). <b>none</b>: only the files and the SUMMARY record.",
            "default": "rows"
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}