Feature/improvements (#26)

- reduce number of log messages
- enable logging to a timestamped file at the root of git repository
- improve plotting
- refactor use of earthaccess
- Discontinue python 3.9
- update github actions; refactor to resolve import problems in python 3.10
---------

Co-authored-by: kaza <kaza@dhigroup.com>

Author: kzalite

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: chartboost/ruff-action@v1
 
   unit-tests:
@@ -20,10 +20,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -56,7 +56,7 @@ jobs:
       vars.RUN_INTEGRATION_TESTS == 'true'
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v5

.gitignore

@@ -142,7 +142,7 @@ dmypy.json
 /data/*
 /quarto/*
 /development/*
-
+/logs/*
 
 # development notebooks
 /notebooks/river_development.ipynb

HydroEO/downloaders/creodias.py

@@ -344,7 +344,7 @@ def _token_age(session_start_time):
         logger.info("Generating session token")
         token = _get_token(username, password)
         session_start_time = time.time()
-    logger.info("Session token age: %.2f minutes", _token_age(session_start_time))
+    logger.debug("Session token age: %.2f minutes", _token_age(session_start_time))
 
     if show_progress:
         if len(uids) > 0:
@@ -357,7 +357,7 @@ def _token_age(session_start_time):
     for uid in uids:
         # assess age of token, (Expires every ten minutes so we refresh every 9 minutes)
         if _token_age(session_start_time) > 9:
-            logger.info(
+            logger.debug(
                 "Session token age: %.2f minutes. Refreshing session token now.",
                 _token_age(session_start_time),
             )

HydroEO/flows.py

@@ -147,7 +147,12 @@ def initialize_rivers(prj: "Project") -> None:
 
     id_label = "node" if prj.rivers.target_id_col == "node_id" else "reach"
     logger.info(
-        "Initialized river %s ids: %s",
+        "Found river %s %s ids",
+        len(prj.rivers.target_ids),
+        id_label,
+    )
+    logger.debug(
+        "Found river %s ids: %s",
         id_label,
         ", ".join(str(target_id) for target_id in prj.rivers.target_ids),
     )
@@ -217,15 +222,15 @@ def _ensure_sword_database(prj: "Project") -> str:
         "SWORD_v17b database not found in %s. Downloading and extracting it now.",
         sword_dir,
     )
-    general.ifnotmakedirs(sword_dir)
+    general.ifnotmakedirs(os.path.join(prj.dirs["main"], "SWORD_v17b_gpkg"))
 
     zip_path = os.path.join(prj.dirs["main"], "SWORD_v17b_gpkg.zip")
     url_request.urlretrieve(SWORD_V17B_ZIP_URL, zip_path)
 
     with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(sword_dir)
+        zip_ref.extractall(os.path.join(prj.dirs["main"], "SWORD_v17b_gpkg"))
 
-    os.remove(zip_path)
+    # os.remove(zip_path)
     return sword_dir
 
 
@@ -484,13 +489,14 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
     waterbody_groups = _group_river_targets_by_waterbody(prj)
 
     summary = {
-        "requested": len(waterbody_groups),
+        "requested": 0,
         "successful": 0,
         "failed": 0,
         "empty_after_filter": 0,
     }
 
     for wb_id, target_ids in waterbody_groups.items():
+        summary["requested"] = summary["requested"] + len(target_ids)
         output_path = os.path.join(
             prj.dirs["swot"], "rivers", str(wb_id), "timeseries.csv"
         )
@@ -501,8 +507,16 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
         if latest_obs is not None:
             wb_startdate = latest_obs
 
+        deferred_warnings = []
+
+        def _defer_warning(message, *args):
+            if args:
+                deferred_warnings.append(message % args)
+            else:
+                deferred_warnings.append(message)
+
         frames = []
-        for target_id in target_ids:
+        for target_id in tqdm(target_ids, desc="Downloading hydrocron data"):
             try:
                 query_params = {
                     "feature": feature,
@@ -520,7 +534,7 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
                     status_code = getattr(response, "status", response.getcode())
                     payload = json.loads(response.read().decode("utf-8"))
             except Exception as exc:
-                logger.warning(
+                _defer_warning(
                     "Hydrocron request failed for %s %s: %s",
                     prj.rivers.target_id_col,
                     target_id,
@@ -535,7 +549,7 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
                 else None
             )
             if status_code != 200 or not csv_payload:
-                logger.warning(
+                _defer_warning(
                     "Hydrocron returned status %s for %s %s",
                     status_code,
                     prj.rivers.target_id_col,
@@ -547,7 +561,7 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
             try:
                 df = pd.read_csv(StringIO(csv_payload))
             except Exception as exc:
-                logger.warning(
+                _defer_warning(
                     "Failed to parse CSV for %s %s: %s",
                     prj.rivers.target_id_col,
                     target_id,
@@ -558,13 +572,13 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
 
             if df.empty or quality_column not in df.columns:
                 if df.empty:
-                    logger.info(
+                    _defer_warning(
                         "Hydrocron returned no data for %s %s",
                         prj.rivers.target_id_col,
                         target_id,
                     )
                 else:
-                    logger.warning(
+                    _defer_warning(
                         "Quality column %s not in Hydrocron response for %s %s",
                         quality_column,
                         prj.rivers.target_id_col,
@@ -574,7 +588,7 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
 
             df = df[df[quality_column] <= max_q]
             if df.empty:
-                logger.info(
+                _defer_warning(
                     "All Hydrocron observations filtered for %s %s (quality > %s)",
                     prj.rivers.target_id_col,
                     target_id,
@@ -590,8 +604,11 @@ def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> N
             combined = pd.concat(frames, ignore_index=True)
             combined.to_csv(output_path, index=False)
 
+        for warning in deferred_warnings:
+            logger.debug(warning)
+
     logger.info(
-        "Hydrocron download complete: %s requested, %s successful, %s failed, %s empty after filtering",
+        "Hydrocron download complete: %s requested, %s successful, %s failed, %s empty after filtering. See file logs for more info.",
         summary["requested"],
         summary["successful"],
         summary["failed"],

HydroEO/logging_config.py

@@ -1,37 +1,81 @@
 """Logging configuration for HydroEO application."""
 
 import logging
+from datetime import datetime
+from pathlib import Path
 
 
-def setup_logging(level=logging.INFO):
-    """Configure logging for HydroEO to display messages in terminal.
+def _ensure_log_dir():
+    """Create logs directory at project root if it doesn't exist.
 
-    Configures the HydroEO logger to output to console with a standard format.
+    Returns
+    -------
+    Path
+        Path to the logs directory.
+    """
+    project_root = Path(
+        __file__
+    ).parent.parent  # HydroEO/logging_config.py -> HydroEO/ -> project root
+    logs_dir = project_root / "logs"
+    logs_dir.mkdir(exist_ok=True)
+    return logs_dir
+
+
+def _get_log_filepath():
+    """Generate a timestamped log file path.
+
+    Returns
+    -------
+    Path
+        Path to the log file with timestamp.
+    """
+    logs_dir = _ensure_log_dir()
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    return logs_dir / f"HydroEO_{timestamp}.log"
+
+
+def setup_logging(level=logging.INFO, enable_file_logging=True):
+    """Configure logging for HydroEO with console and optional file output.
+
+    Configures the HydroEO logger to output to console at the specified level.
+    If file logging is enabled, also logs to a timestamped file at DEBUG level.
     This should be called at application startup before running any HydroEO operations.
 
     Parameters
     ----------
     level : int
-        Logging level (logging.DEBUG, logging.INFO, logging.WARNING, etc.)
+        Logging level for console output (logging.DEBUG, logging.INFO, logging.WARNING, etc.)
         Default is logging.INFO.
+    enable_file_logging : bool
+        If True, also logs to a timestamped file at DEBUG level.
+        Default is True.
 
     Examples
     --------
     >>> from HydroEO.logging_config import setup_logging
-    >>> setup_logging(logging.DEBUG)  # Enable debug messages
+    >>> setup_logging(logging.DEBUG)  # Console at DEBUG, file at DEBUG
+    >>> setup_logging(logging.INFO, enable_file_logging=False)  # Console only
     """
     logger = logging.getLogger("HydroEO")
-    logger.setLevel(level)
+    logger.setLevel(logging.DEBUG)  # Logger itself should be at DEBUG to capture all
+
+    # Create formatter
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
 
     # Create console handler if one doesn't exist
     if not logger.handlers:
-        handler = logging.StreamHandler()
-        handler.setLevel(level)
-
-        # Create formatter
-        formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-        handler.setFormatter(formatter)
-        logger.addHandler(handler)
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(level)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+        # Create file handler if enabled
+        if enable_file_logging:
+            log_filepath = _get_log_filepath()
+            file_handler = logging.FileHandler(log_filepath)
+            file_handler.setLevel(logging.DEBUG)
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)