manifest.yaml

version: "4.3.0"
type: DeclarativeSource

description: >
  DataCops source connector. Syncs fraud detection data — identified users,
  sessions, bot detections, and VPN/proxy detections — from DataCops into
  any data warehouse supported by Airbyte.

check:
  type: CheckStream
  stream_names:
    - identified_users

definitions:
  # ── Shared base requester ────────────────────────────────────────────────
  base_requester:
    type: HttpRequester
    url_base: "{{ config['base_url'] }}"
    http_method: GET
    authenticator:
      type: ApiKeyAuthenticator
      header: "x-dc-key"
      api_token: "{{ config['api_key'] }}"

  # ── Cursor-based pagination ──────────────────────────────────────────────
  # Each response includes pagination.next_since; we pass it as `since` in the
  # next request until has_more is false.
  cursor_paginator:
    type: DefaultPaginator
    pagination_strategy:
      type: CursorPagination
      cursor_value: "{{ response.pagination.next_since }}"
      stop_condition: "{{ response.pagination.has_more == false }}"
      page_size: 100
    page_token_option:
      type: RequestOption
      inject_into: request_parameter
      field_name: since

  # ── Record selector — data is in response.data[] ────────────────────────
  records_selector:
    type: RecordSelector
    extractor:
      type: DpathExtractor
      field_path:
        - data

  # ── Incremental cursor — use identified_at / detected_at / created_at ───
  datetime_cursor:
    type: DatetimeBasedCursor
    cursor_field: "{{ parameters.cursor_field }}"
    datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
    cursor_granularity: PT1S
    start_datetime:
      type: MinMaxDatetime
      datetime: "{{ config.get('start_date', '2020-01-01T00:00:00Z') }}"
      datetime_format: "%Y-%m-%dT%H:%M:%SZ"
      min_datetime: "2020-01-01T00:00:00Z"
    end_datetime:
      type: MinMaxDatetime
      datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}"
      datetime_format: "%Y-%m-%dT%H:%M:%SZ"
    step: P30D
    start_time_option:
      type: RequestOption
      inject_into: request_parameter
      field_name: since
    end_time_option:
      type: RequestOption
      inject_into: request_parameter
      field_name: until

streams:
  # ── identified_users ─────────────────────────────────────────────────────
  - type: DeclarativeStream
    name: identified_users
    primary_key: id
    $parameters:
      cursor_field: identified_at
    schema_loader:
      type: InlineSchemaLoader
      schema:
        type: object
        properties:
          id:                    { type: [string, "null"] }
          app_id:                { type: [string, "null"] }
          email:                 { type: [string, "null"] }
          normalized_email:      { type: [string, "null"] }
          session_id:            { type: [string, "null"] }
          user_id:               { type: [string, "null"] }
          page_url:              { type: [string, "null"] }
          user_agent:            { type: [string, "null"] }
          source:                { type: [string, "null"] }
          fingerprint_hash:      { type: [string, "null"] }
          fingerprint_confidence: { type: [number, "null"] }
          email_risk_score:      { type: [number, "null"] }
          email_risk_label:      { type: [string, "null"] }
          is_disposable:         { type: [boolean, "null"] }
          is_free_provider:      { type: [boolean, "null"] }
          is_gibberish:          { type: [boolean, "null"] }
          is_bulk_pattern:       { type: [boolean, "null"] }
          ai_fake_score:         { type: [number, "null"] }
          ai_verdict:            { type: [string, "null"] }
          ip_address:            { type: [string, "null"] }
          ip_country:            { type: [string, "null"] }
          ip_is_vpn:             { type: [boolean, "null"] }
          ip_is_proxy:           { type: [boolean, "null"] }
          ip_is_tor:             { type: [boolean, "null"] }
          ip_risk_score:         { type: [number, "null"] }
          identified_at:         { type: [string, "null"], format: date-time }
          created_at:            { type: [string, "null"], format: date-time }
    retriever:
      type: SimpleRetriever
      requester:
        $ref: "#/definitions/base_requester"
        path: "/api/v1/export/identified-users"
        request_parameters:
          limit: "100"
      record_selector:
        $ref: "#/definitions/records_selector"
      paginator:
        $ref: "#/definitions/cursor_paginator"
    incremental_sync:
      $ref: "#/definitions/datetime_cursor"
      $parameters:
        cursor_field: identified_at

  # ── sessions ─────────────────────────────────────────────────────────────
  - type: DeclarativeStream
    name: sessions
    primary_key: id
    $parameters:
      cursor_field: start_time
    schema_loader:
      type: InlineSchemaLoader
      schema:
        type: object
        properties:
          id:           { type: [string, "null"] }
          session_id:   { type: [string, "null"] }
          app_id:       { type: [string, "null"] }
          user_id:      { type: [string, "null"] }
          start_time:   { type: [string, "null"], format: date-time }
          end_time:     { type: [string, "null"], format: date-time }
          duration_ms:  { type: [number, "null"] }
          page_count:   { type: [integer, "null"] }
          country:      { type: [string, "null"] }
          city:         { type: [string, "null"] }
          ip_address:   { type: [string, "null"] }
          is_vpn:       { type: [boolean, "null"] }
          is_proxy:     { type: [boolean, "null"] }
          is_tor:       { type: [boolean, "null"] }
          user_agent:   { type: [string, "null"] }
          platform:     { type: [string, "null"] }
          is_bot:       { type: [boolean, "null"] }
          bot_reasons:  { type: [string, "null"] }
          referrer:     { type: [string, "null"] }
          landing_url:  { type: [string, "null"] }
          created_at:   { type: [string, "null"], format: date-time }
    retriever:
      type: SimpleRetriever
      requester:
        $ref: "#/definitions/base_requester"
        path: "/api/v1/export/sessions"
        request_parameters:
          limit: "100"
      record_selector:
        $ref: "#/definitions/records_selector"
      paginator:
        $ref: "#/definitions/cursor_paginator"
    incremental_sync:
      $ref: "#/definitions/datetime_cursor"
      $parameters:
        cursor_field: start_time

  # ── bot_detections ───────────────────────────────────────────────────────
  - type: DeclarativeStream
    name: bot_detections
    primary_key: id
    $parameters:
      cursor_field: detected_at
    schema_loader:
      type: InlineSchemaLoader
      schema:
        type: object
        properties:
          id:          { type: [string, "null"] }
          session_id:  { type: [string, "null"] }
          app_id:      { type: [string, "null"] }
          user_id:     { type: [string, "null"] }
          bot_reasons: { type: [string, "null"] }
          user_agent:  { type: [string, "null"] }
          country:     { type: [string, "null"] }
          ip_address:  { type: [string, "null"] }
          is_vpn:      { type: [boolean, "null"] }
          is_proxy:    { type: [boolean, "null"] }
          detected_at: { type: [string, "null"], format: date-time }
          created_at:  { type: [string, "null"], format: date-time }
    retriever:
      type: SimpleRetriever
      requester:
        $ref: "#/definitions/base_requester"
        path: "/api/v1/export/bot-detections"
        request_parameters:
          limit: "100"
      record_selector:
        $ref: "#/definitions/records_selector"
      paginator:
        $ref: "#/definitions/cursor_paginator"
    incremental_sync:
      $ref: "#/definitions/datetime_cursor"
      $parameters:
        cursor_field: detected_at

  # ── vpn_proxy_detections ─────────────────────────────────────────────────
  - type: DeclarativeStream
    name: vpn_proxy_detections
    primary_key: id
    $parameters:
      cursor_field: detected_at
    schema_loader:
      type: InlineSchemaLoader
      schema:
        type: object
        properties:
          id:           { type: [string, "null"] }
          session_id:   { type: [string, "null"] }
          app_id:       { type: [string, "null"] }
          user_id:      { type: [string, "null"] }
          ip_address:   { type: [string, "null"] }
          is_vpn:       { type: [boolean, "null"] }
          is_proxy:     { type: [boolean, "null"] }
          is_tor:       { type: [boolean, "null"] }
          isp:          { type: [string, "null"] }
          organization: { type: [string, "null"] }
          country:      { type: [string, "null"] }
          detected_at:  { type: [string, "null"], format: date-time }
          created_at:   { type: [string, "null"], format: date-time }
    retriever:
      type: SimpleRetriever
      requester:
        $ref: "#/definitions/base_requester"
        path: "/api/v1/export/vpn-proxy-detections"
        request_parameters:
          limit: "100"
      record_selector:
        $ref: "#/definitions/records_selector"
      paginator:
        $ref: "#/definitions/cursor_paginator"
    incremental_sync:
      $ref: "#/definitions/datetime_cursor"
      $parameters:
        cursor_field: detected_at

spec:
  type: Spec
  connection_specification:
    type: object
    title: DataCops Source Spec
    required:
      - api_key
      - base_url
    additionalProperties: true
    properties:
      api_key:
        type: string
        title: Private API Key
        description: >
          Your DataCops Private API Key (starts with dcp_). Generate one in
          your DataCops dashboard under Settings → API Keys → Private Keys.
        airbyte_secret: true
        order: 0
      base_url:
        type: string
        title: API Base URL
        description: Your DataCops API base URL.
        default: "https://api.joindatacops.com"
        examples:
          - "https://api.joindatacops.com"
        order: 1
      start_date:
        type: string
        title: Start Date
        description: >
          UTC start date for the initial sync in ISO 8601 format.
          Records created before this date will not be synced.
        pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
        examples:
          - "2024-01-01T00:00:00Z"
        order: 2