{
  "openapi": "3.0.1",
  "info": {
    "title": "Dedupe and Merge Datasets: Only New Items",
    "description": "Merge Apify datasets, runs and JSON or CSV files, remove duplicates by key fields with URL, email, phone and fuzzy matching, and output only items not seen in earlier runs. Runs after any scraper as an integration.",
    "version": "0.1",
    "x-build-id": "uU2dIfQkmJ1cRG3fx"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/fernandosord~dataset-merge-dedupe/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-fernandosord-dataset-merge-dedupe",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/fernandosord~dataset-merge-dedupe/runs": {
      "post": {
        "operationId": "runs-sync-fernandosord-dataset-merge-dedupe",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/fernandosord~dataset-merge-dedupe/run-sync": {
      "post": {
        "operationId": "run-sync-fernandosord-dataset-merge-dedupe",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "datasetIds": {
            "title": "Datasets",
            "type": "array",
            "description": "Apify datasets to merge and dedupe, in the order given. Pick them, or paste dataset IDs. To run this Actor automatically after a scraper, add it as an integration and set this field to <code>[\"{{resource.defaultDatasetId}}\"]</code> (see the README)."
          },
          "runIds": {
            "title": "Run IDs",
            "type": "array",
            "description": "Actor run IDs whose default dataset should be read. Apify's limited permissions may block looking up runs; if a run is refused, paste its dataset ID into \"Datasets\" instead.",
            "items": {
              "type": "string"
            }
          },
          "dataUrls": {
            "title": "JSON or CSV URLs",
            "type": "array",
            "description": "Public http or https URLs of JSON arrays, JSON Lines or CSV files (comma, semicolon, tab or pipe separated, first row = header), up to 50 MB each and 15% of the run memory in total (about 150 MB at the default 1 GB). Private and local network addresses are refused.",
            "items": {
              "type": "string"
            }
          },
          "rawData": {
            "title": "Inline JSON data",
            "type": "array",
            "description": "Up to 10,000 items as a JSON array of objects. Useful for trying the Actor or for small lists."
          },
          "compareDatasetIds": {
            "title": "Compare against datasets",
            "type": "array",
            "description": "Optional. Records from these datasets are never output; records in \"Datasets\" that match one of them by key are removed. Use it to drop leads you already have, or items from an older export."
          },
          "dedupeFields": {
            "title": "Dedupe key fields",
            "type": "array",
            "description": "Records whose values in all these fields match (after normalisation) are duplicates. Use dot paths for nested fields, for example <code>address.city</code>. Leave empty to remove only records that are identical in every field.",
            "items": {
              "type": "string"
            }
          },
          "urlFields": {
            "title": "Treat as URLs",
            "type": "array",
            "description": "Key fields that hold web addresses. They are compared without http/https, <code>www.</code>, trailing slash, <code>#fragment</code>, default port and tracking parameters (<code>utm_*</code>, <code>gclid</code>, <code>fbclid</code>, <code>msclkid</code> and similar). Other query parameters are kept and sorted.",
            "items": {
              "type": "string"
            }
          },
          "emailFields": {
            "title": "Treat as emails",
            "type": "array",
            "description": "Key fields that hold email addresses. They are lowercased, and a <code>mailto:</code> prefix or a display name such as <code>Jane &lt;jane@x.com&gt;</code> is removed.",
            "items": {
              "type": "string"
            }
          },
          "phoneFields": {
            "title": "Treat as phone numbers",
            "type": "array",
            "description": "Key fields that hold phone numbers. They are converted to international format (for example <code>+14155552671</code>) with libphonenumber, so <code>(415) 555-2671</code> and <code>+1 415 555 2671</code> match.",
            "items": {
              "type": "string"
            }
          },
          "defaultPhoneCountry": {
            "title": "Default phone country",
            "pattern": "^([A-Za-z]{2})?$",
            "type": "string",
            "description": "Two-letter country code (US, GB, DE, BR...) for phone numbers written without a + country code. Without it, such numbers are compared by their digits only."
          },
          "caseSensitive": {
            "title": "Case-sensitive keys",
            "type": "boolean",
            "description": "Off: \"ACME\" and \"acme\" match. On: they don't. Spaces at the ends and repeated spaces are ignored either way.",
            "default": false
          },
          "fuzzyField": {
            "title": "Fuzzy match field",
            "type": "string",
            "description": "Optional text field, such as a company or product name, compared by similarity instead of exact value. Case, accents, punctuation and word order are ignored. A record is compared with earlier records that share at least one word with it and have the same values in the other key fields. Leave empty for exact matching only."
          },
          "fuzzyThreshold": {
            "title": "Fuzzy similarity threshold",
            "minimum": 50,
            "maximum": 100,
            "type": "integer",
            "description": "How similar two texts must be to count as the same, from 50 to 100. Similarity is 100 minus the edit distance as a percentage of the longer text: \"Acme Corporation\" and \"Acme Corporaton\" are 94% similar.",
            "default": 90
          },
          "keep": {
            "title": "Which record to keep",
            "enum": [
              "first",
              "last",
              "most-complete",
              "merge"
            ],
            "type": "string",
            "description": "<b>First</b> keeps the first record of each group and is the fastest. <b>Last</b> keeps the latest one. <b>Most complete</b> keeps the record with the most filled-in fields. <b>Merge</b> starts from the first record and fills its empty fields from the later duplicates. All but First read the input twice (inputs up to 50,000 items are kept in memory instead).",
            "default": "first"
          },
          "missingKeyAction": {
            "title": "Records with an empty key field",
            "enum": [
              "keep",
              "drop",
              "compare-as-empty"
            ],
            "type": "string",
            "description": "What to do with records where a key field is missing or empty. <b>Keep</b> outputs each of them (they never count as duplicates). <b>Drop</b> leaves them out. <b>Compare as empty</b> treats the empty value as a value, so two records missing the same field can match.",
            "default": "keep"
          },
          "onlyNewItems": {
            "title": "Only output items not seen in earlier runs",
            "type": "boolean",
            "description": "Remembers the key of every item this Actor outputs, under the state key below, and leaves out items with a key it has seen before. Made for scheduled scrapers: each run returns only what's new.",
            "default": false
          },
          "stateKey": {
            "title": "State key",
            "type": "string",
            "description": "Remembered items are kept per state key. Use a different key for each scraper or task, so separate lists don't mix. A new key starts empty.",
            "default": "default"
          },
          "outputChanged": {
            "title": "Also output changed items",
            "type": "boolean",
            "description": "Also output items seen before whose other fields changed since they were last output (for example a new price). Compared by a hash of every field except the key fields and the fields below.",
            "default": false
          },
          "changeIgnoreFields": {
            "title": "Fields to ignore when checking for changes",
            "type": "array",
            "description": "Fields that change on every scrape and should not count as a change, such as <code>scrapedAt</code> or <code>position</code>.",
            "items": {
              "type": "string"
            }
          },
          "rememberItems": {
            "title": "Remember items from this run",
            "type": "boolean",
            "description": "Turn off to test settings: the run shows what is new but doesn't update the remembered items.",
            "default": true
          },
          "forgetState": {
            "title": "Forget remembered items first",
            "type": "boolean",
            "description": "Clears the items remembered under this state key before the run, so every item counts as new. Turn it off again after one run.",
            "default": false
          },
          "rememberMaxKeys": {
            "title": "Maximum remembered items",
            "minimum": 1000,
            "maximum": 10000000,
            "type": "integer",
            "description": "When more keys than this are remembered, the ones not seen for the longest time are forgotten. Each remembered item takes 20 bytes of storage and roughly 200 bytes of memory during a run, so raise the run memory for more than 1 million.",
            "default": 1000000
          },
          "outputItems": {
            "title": "What to output",
            "enum": [
              "unique",
              "duplicates",
              "nothing"
            ],
            "type": "string",
            "description": "<b>Unique items</b>: one record per key. <b>Duplicates</b>: the records that were removed. <b>Nothing</b>: only the counts in the OUTPUT summary.",
            "default": "unique"
          },
          "outputDataset": {
            "title": "Output dataset name",
            "type": "string",
            "description": "Optional. Write to a named dataset instead of the run's default dataset. It is created on the first run, and later runs add to it. Letters, digits and hyphens only."
          },
          "outputFields": {
            "title": "Output fields",
            "type": "array",
            "description": "Optional. Only these fields are written (dot paths allowed). Leave empty to keep all fields. Choosing fields also makes datasets faster to read.",
            "items": {
              "type": "string"
            }
          },
          "omitFields": {
            "title": "Fields to remove",
            "type": "array",
            "description": "Optional. Fields removed from every written record.",
            "items": {
              "type": "string"
            }
          },
          "renameFields": {
            "title": "Rename fields",
            "type": "object",
            "description": "Optional JSON object from old to new name, for example <code>{\"title\": \"name\", \"address.city\": \"city\"}</code>. Renaming happens after field selection."
          },
          "writeNormalizedValues": {
            "title": "Write cleaned key values",
            "type": "boolean",
            "description": "Replace key field values in the output with their normalised form: URLs without tracking parameters, lowercased emails, phone numbers in +international format.",
            "default": false
          },
          "addStatusField": {
            "title": "Add a _status field",
            "type": "boolean",
            "description": "Adds <code>_status</code> to each written record: <code>new</code> or <code>changed</code> in only-new mode, <code>unique</code> otherwise, <code>duplicate</code> when outputting duplicates.",
            "default": false
          },
          "addSourceField": {
            "title": "Add a _source field",
            "type": "boolean",
            "description": "Adds <code>_source</code> with the dataset ID or URL each record came from.",
            "default": false
          },
          "filters": {
            "title": "Filters",
            "type": "array",
            "description": "Optional. Only records that pass are deduped and output. JSON array such as <code>[{\"field\": \"price\", \"op\": \"less-than\", \"value\": 50}, {\"field\": \"email\", \"op\": \"exists\"}]</code>. Ops: equals, not-equals, contains, not-contains, starts-with, ends-with, greater-than, greater-or-equal, less-than, less-or-equal, exists, not-exists, matches-regex. Text comparisons ignore case."
          },
          "filterMatch": {
            "title": "Records must pass",
            "enum": [
              "all",
              "any"
            ],
            "type": "string",
            "description": "Whether a record must pass all filters or at least one.",
            "default": "all"
          },
          "offset": {
            "title": "Skip the first items",
            "minimum": 0,
            "type": "integer",
            "description": "Number of input items to skip, counted across all inputs in order.",
            "default": 0
          },
          "limit": {
            "title": "Maximum items to read",
            "minimum": 1,
            "type": "integer",
            "description": "Stop after reading this many input items. Leave empty for no limit."
          },
          "payload": {
            "title": "Integration payload",
            "type": "object",
            "description": "Filled in by Apify integrations and webhooks. Its <code>resource.defaultDatasetId</code> is added to \"Datasets\"."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}