[AIMOD-1383] Engagement GCS Artifacts for merino use daily-country propensity (#9702)

* saves a daily snapshot of propensity

* newtab_merino_propensity_v2

* trying to minimize diff with v1

* added country

* add new v4 instead of using v3 for merino data export

* use country specific propensities

* change type

* closer to v3

* clashing withv v3?

* Use propensity v2 for Merino newtab extract

* added IT region

Author: mlplyler

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_merino_extract_v3/metadata.yaml

@@ -1,6 +1,6 @@
 friendly_name: Newtab Merino Extract
 description: |-
-  Aggregated Newtab events for Merino recommendations.
+  Aggregated Newtab events for Merino recommendations using v2 propensity weights.
   See https://mozilla-hub.atlassian.net/browse/MC-1256
 owners:
 - mmiermans@mozilla.com

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_merino_extract_v3/query.sql

@@ -69,30 +69,60 @@ raw_grouped_totals AS (
     format,
     section_position
 ),
+propensity_weights AS (
+  SELECT
+    country,
+    position,
+    tile_format,
+    weight
+  FROM
+    `moz-fx-data-shared-prod.telemetry_derived.newtab_merino_propensity_v2`
+  WHERE
+    layout = 'SECTION_GRID'
+    AND section_position IS NULL
+    AND snapshot_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 14 DAY)
+  QUALIFY
+    snapshot_date = MAX(snapshot_date) OVER ()
+),
 /* Separate and adjust section events */
 section_events AS (
   SELECT
     rw.normalized_country_code,
     rw.corpus_item_id,
     rw.raw_impression_count,
     -- apply propensity scaling to impressions only
+    -- prefer exact-format weights, then fall back to 'any' format weights
     rw.raw_impression_count / COALESCE(
-      wt_exact.weight,
-      wt_any.weight,
+      wt_country_exact.weight,
+      wt_global_exact.weight,
+      wt_country_any.weight,
+      wt_global_any.weight,
       1.0
     ) AS adjusted_impression_count,
     rw.report_count,
     rw.click_count
   FROM
     raw_grouped_totals rw
   LEFT JOIN
-    `moz-fx-data-shared-prod.telemetry_derived.newtab_merino_propensity_v1` wt_exact
-    ON SAFE_CAST(wt_exact.position AS INT64) = rw.position
-    AND wt_exact.tile_format = rw.format
+    propensity_weights wt_country_exact
+    ON wt_country_exact.country = rw.normalized_country_code
+    AND SAFE_CAST(wt_country_exact.position AS INT64) = rw.position
+    AND wt_country_exact.tile_format = rw.format
+  LEFT JOIN
+    propensity_weights wt_country_any
+    ON wt_country_any.country = rw.normalized_country_code
+    AND SAFE_CAST(wt_country_any.position AS INT64) = rw.position
+    AND wt_country_any.tile_format = 'any'
+  LEFT JOIN
+    propensity_weights wt_global_exact
+    ON wt_global_exact.country IS NULL
+    AND SAFE_CAST(wt_global_exact.position AS INT64) = rw.position
+    AND wt_global_exact.tile_format = rw.format
   LEFT JOIN
-    `moz-fx-data-shared-prod.telemetry_derived.newtab_merino_propensity_v1` wt_any
-    ON SAFE_CAST(wt_any.position AS INT64) = rw.position
-    AND wt_any.tile_format = 'any'
+    propensity_weights wt_global_any
+    ON wt_global_any.country IS NULL
+    AND SAFE_CAST(wt_global_any.position AS INT64) = rw.position
+    AND wt_global_any.tile_format = 'any'
   WHERE
     rw.section_position IS NOT NULL
 ),

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_merino_priors_v1/query.sql

@@ -15,7 +15,7 @@ target_regions AS (
   SELECT
     region
   FROM
-    UNNEST(['US', 'CA', 'DE', 'CH', 'AT', 'GB', 'IE', 'BE', 'PL', 'FR', 'ES']) AS region
+    UNNEST(['US', 'CA', 'DE', 'CH', 'AT', 'GB', 'IE', 'BE', 'PL', 'FR', 'ES', 'IT']) AS region
 ),
 corpus_items AS (
   SELECT DISTINCT
@@ -231,7 +231,7 @@ SELECT
 FROM
   per_region_final
 WHERE
-  region IN ('US', 'CA', 'DE', 'CH', 'AT', 'GB', 'IE', 'BE', 'PL', 'FR', 'ES')
+  region IN ('US', 'CA', 'DE', 'CH', 'AT', 'GB', 'IE', 'BE', 'PL', 'FR', 'ES', 'IT')
 UNION ALL
 SELECT
   region,