add city full

Author: RemDelaporteMathurin

src/data_fetching/fusion_energy_base.py

@@ -30,4 +30,22 @@ def aggregate_data(organization_generator):
     all_data = []
     for data in organization_generator:
         all_data.extend(data["results"])
-    return pd.DataFrame(all_data)
+
+    df = pd.DataFrame(all_data)
+
+    df["city_full"] = df.apply(
+        lambda row: (
+            float("nan")
+            if pd.isna(row["city"])
+            else ", ".join(
+                [
+                    str(x)
+                    for x in [row.get("city"), row.get("region"), row.get("country")]
+                    if pd.notna(x) and str(x).strip()
+                ]
+            )
+        ),
+        axis=1,
+    )
+
+    return df

src/data_fetching/google_analytics.py

@@ -35,6 +35,7 @@ def run_report(
     dimensions = [Dimension(name="date")] if daily else []
     dimensions += [
         Dimension(name="city"),
+        Dimension(name="region"),
         Dimension(name="country"),
     ]
 

src/geospatial/geocoding.py

@@ -11,11 +11,11 @@ def find_locations_from_cache(df, filename: str, update_cache=True):
         print(f"No cache found at {filename}, starting with an empty cache.")
         cache = gpd.GeoDataFrame(columns=["city", "geometry"], crs="EPSG:4326")
 
-    cities_found = set(df["city"]) & set(cache["city"])
+    cities_found = set(df["city_full"]) & set(cache["city"])
     locations_found = cache[cache["city"].isin(cities_found)]
 
     # for cities not found in cache, geocode and add to cache
-    locs_not_found = set(df["city"]) - set(cache["city"])
+    locs_not_found = set(df["city_full"]) - set(cache["city"])
     if locs_not_found:
         print(f"Locations not found in cache: {locs_not_found}")
         locs_new = gpd.tools.geocode(list(locs_not_found))
@@ -28,8 +28,8 @@ def find_locations_from_cache(df, filename: str, update_cache=True):
     )
 
     # ensure locations matches the order of df and avoid dtype mismatch errors
-    merge_df = df[["city"]].copy()
-    merge_df["city"] = merge_df["city"].astype(str)
+    merge_df = df[["city_full"]].copy()
+    merge_df["city"] = merge_df["city_full"].astype(str)
     locations["city"] = locations["city"].astype(str)
     locations = merge_df.merge(locations, on="city", how="left")
     locations = gpd.GeoDataFrame(locations, geometry="geometry")