Merge pull request #47 from SeaCelo/feature/port-ogzaf-106-111

Port calibration API structure and offline gating

Author: marcelolafleur

.github/workflows/deploy_docs.yml

@@ -34,8 +34,8 @@ jobs:
       - name: Build # Build Jupyter Book
         shell: bash -l {0}
         run: |
-          pip install jupyter-book>=0.11.3
-          pip install sphinxcontrib-bibtex>=2.0.0
+          pip install "jupyter-book<2.0.0"
+          pip install "sphinxcontrib-bibtex>=2.0.0"
           pip install -e .
           python -m ipykernel install --user --name=ogphl-dev
           jb build ./docs/book

.github/workflows/docs_check.yml

@@ -32,8 +32,8 @@ jobs:
       - name: Build # Build Jupyter Book
         shell: bash -l {0}
         run: |
-          pip install jupyter-book>=0.11.3
-          pip install sphinxcontrib-bibtex>=2.0.0
+          pip install "jupyter-book<2.0.0"
+          pip install "sphinxcontrib-bibtex>=2.0.0"
           pip install -e .
           python -m ipykernel install --user --name=ogphl-dev
           jb build ./docs/book

README.md

@@ -49,10 +49,12 @@ Once the package is installed, one can adjust parameters in the OG-Core `Specifi
 ```
 from ogcore.parameters import Specifications
 from ogphl.calibrate import Calibration
+from ogphl.utils import is_connected
 p = Specifications()
-c = Calibration(p)
-updated_params = c.get_dict()
-p.update_specifications({'initial_debt_ratio': updated_params['initial_debt_ratio']})
+if is_connected():
+    c = Calibration(p, update_from_api=True)
+    updated_params = c.get_dict()
+    p.update_specifications(updated_params)
 ```
 
 ## Disclaimer

docs/book/content/UNtutorial/solutions/3perOG/SS.py

@@ -28,7 +28,6 @@
 from matplotlib.ticker import MultipleLocator
 import os
 
-
 # Define functions
 
 

docs/create_doc_figures.py

@@ -14,7 +14,6 @@
 from ogcore import demographics as demog
 import ogphl
 
-
 CUR_DIR = os.path.dirname(os.path.realpath(__file__))
 UN_COUNTRY_CODE = "608"
 plot_path = os.path.join(CUR_DIR, "book", "content", "calibration", "images")

environment.yml

@@ -34,7 +34,6 @@ dependencies:
 - pip
 - pip:
   - openpyxl>=3.1.2
-  - pandas-datareader
   - linecheck
   - ogcore>=0.14.5
   - sphinx-exercise

examples/run_og_phl.py

@@ -54,7 +54,7 @@ def main():
     p.update_specifications(defaults)
     # Update parameters from calibrate.py Calibration class
     if is_connected():  # only update if connected to internet
-        c = Calibration(p)
+        c = Calibration(p, update_from_api=True)
         updated_params = c.get_dict()
         p.update_specifications(updated_params)
 

ogphl/calibrate.py

@@ -1,9 +1,9 @@
 from ogphl import macro_params, income
 from ogphl import input_output as io
 import os
+import warnings
 import numpy as np
 import datetime
-from ogcore import demographics
 
 UN_COUNTRY_CODE = "608"
 
@@ -18,14 +18,19 @@ def __init__(
         macro_data_end_year=datetime.datetime(2023, 1, 1),
         demographic_data_path=None,
         output_path=None,
+        update_from_api=False,
     ):
         """
         Constructor for the Calibration class.
 
         Args:
             p (OG-Core Specifications object): model parameters
+            macro_data_start_year (datetime): start date for macro data
+            macro_data_end_year (datetime): end date for macro data
             demographic_data_path (str): path to save demographic data
             output_path (str): path to save output to
+            update_from_api (bool): Set True to pull updated data from
+                online sources
 
         Returns:
             None
@@ -36,70 +41,100 @@ def __init__(
             if not os.path.exists(output_path):
                 os.makedirs(output_path)
 
+        # Initialize with overlay values only. Existing values on p are the
+        # baseline from the packaged defaults.
+        self.macro_params = {}
+        self.demographic_params = {}
+        self.e = None
+        self.alpha_c = np.array([1.0]) if p.I == 1 else None
+        self.io_matrix = np.array([[1.0]]) if p.M == 1 else None
+
+        if not update_from_api:
+            return
+
         # Macro estimation
-        self.macro_params = macro_params.get_macro_params(
-            macro_data_start_year, macro_data_end_year
-        )
+        try:
+            self.macro_params = macro_params.get_macro_params(
+                macro_data_start_year,
+                macro_data_end_year,
+                update_from_api=update_from_api,
+            )
+        except Exception as exc:
+            warnings.warn(f"Macro params update failed: {exc}", stacklevel=2)
 
         # io matrix and alpha_c
-        if p.I > 1:  # no need if just one consumption good
-            alpha_c_dict = io.get_alpha_c()
-            # check that model dimensions are consistent with alpha_c
-            assert p.I == len(list(alpha_c_dict.keys()))
-            self.alpha_c = np.array(list(alpha_c_dict.values()))
-        else:
-            self.alpha_c = np.array([1.0])
-        if p.M > 1:  # no need if just one production good
-            io_df = io.get_io_matrix()
-            # check that model dimensions are consistent with io_matrix
-            assert p.M == len(list(io_df.keys()))
-            self.io_matrix = io_df.values
-        else:
-            self.io_matrix = np.array([[1.0]])
-
-        # demographics
-        self.demographic_params = demographics.get_pop_objs(
-            p.E,
-            p.S,
-            p.T,
-            0,
-            99,
-            country_id=UN_COUNTRY_CODE,
-            initial_data_year=p.start_year - 1,
-            final_data_year=p.start_year + 1,
-            GraphDiag=False,
-            download_path=demographic_data_path,
-        )
-
-        # demographics for 80 period lives (needed for getting e below)
-        demog80 = demographics.get_pop_objs(
-            20,
-            80,
-            p.T,
-            0,
-            99,
-            country_id=UN_COUNTRY_CODE,
-            initial_data_year=p.start_year - 1,
-            final_data_year=p.start_year + 1,
-            GraphDiag=False,
-        )
-
-        # earnings profiles
-        self.e = income.get_e_interp(
-            p.E,
-            p.S,
-            p.J,
-            p.lambdas,
-            demog80["omega_SS"],
-        )
+        if p.I > 1:
+            try:
+                alpha_c_dict = io.get_alpha_c()
+                # check that model dimensions are consistent with alpha_c
+                assert p.I == len(list(alpha_c_dict.keys()))
+                self.alpha_c = np.array(list(alpha_c_dict.values()))
+            except Exception as exc:
+                warnings.warn(f"alpha_c update failed: {exc}", stacklevel=2)
+        if p.M > 1:
+            try:
+                io_df = io.get_io_matrix()
+                # check that model dimensions are consistent with io_matrix
+                assert p.M == len(list(io_df.keys()))
+                self.io_matrix = io_df.values
+            except Exception as exc:
+                warnings.warn(f"io_matrix update failed: {exc}", stacklevel=2)
+
+        # Demographics and income are atomic because e depends on demography.
+        try:
+            from ogcore import demographics
+
+            self.demographic_params = demographics.get_pop_objs(
+                p.E,
+                p.S,
+                p.T,
+                0,
+                99,
+                country_id=UN_COUNTRY_CODE,
+                initial_data_year=p.start_year - 1,
+                final_data_year=p.start_year + 1,
+                GraphDiag=False,
+                download_path=demographic_data_path,
+            )
+
+            # demographics for 80 period lives (needed for getting e below)
+            demog80 = demographics.get_pop_objs(
+                20,
+                80,
+                p.T,
+                0,
+                99,
+                country_id=UN_COUNTRY_CODE,
+                initial_data_year=p.start_year - 1,
+                final_data_year=p.start_year + 1,
+                GraphDiag=False,
+            )
+
+            # earnings profiles
+            self.e = income.get_e_interp(
+                p.E,
+                p.S,
+                p.J,
+                p.lambdas,
+                demog80["omega_SS"],
+            )
+        except Exception as exc:
+            warnings.warn(
+                f"Demographics/income update failed: {exc}", stacklevel=2
+            )
+            self.demographic_params = {}
+            self.e = None
 
     # method to return all newly calibrated parameters in a dictionary
     def get_dict(self):
-        dict = {}
-        dict.update(self.macro_params)
-        dict["e"] = self.e
-        dict["alpha_c"] = self.alpha_c
-        dict["io_matrix"] = self.io_matrix
-        dict.update(self.demographic_params)
-
-        return dict
+        d = {}
+        d.update(self.macro_params)
+        d.update(self.demographic_params)
+        if self.e is not None:
+            d["e"] = self.e
+        if self.alpha_c is not None:
+            d["alpha_c"] = self.alpha_c
+        if self.io_matrix is not None:
+            d["io_matrix"] = self.io_matrix
+
+        return d

ogphl/input_output.py

@@ -3,20 +3,30 @@
 import os
 from ogphl.constants import CONS_DICT, PROD_DICT
 
-
 CUR_DIR = os.path.dirname(os.path.realpath(__file__))
 """
 Read in Social Accounting Matrix (SAM) file
 """
-# Read in SAM file
-# SAM file:
 sam_path = os.path.join(CUR_DIR, "data", "002_IFPRI_SAM_PHL_2018_SAM.csv")
-SAM = pd.read_csv(sam_path, index_col=1, thousands=",")
-# replace NaN with 0
-SAM.fillna(0, inplace=True)
 
 
-def get_alpha_c(sam=SAM, cons_dict=CONS_DICT):
+def read_SAM():
+    """
+    Read in the packaged Social Accounting Matrix (SAM) file.
+
+    Returns:
+        SAM (pd.DataFrame | None): Social Accounting Matrix, or None if unavailable
+    """
+    try:
+        sam = pd.read_csv(sam_path, index_col=1, thousands=",")
+        sam.fillna(0, inplace=True)
+        return sam
+    except Exception as exc:
+        print(f"Failed to read packaged SAM file: {exc}")
+        return None
+
+
+def get_alpha_c(sam=None, cons_dict=CONS_DICT):
     """
     Calibrate the alpha_c vector, showing the shares of household
     expenditures for each consumption category
@@ -28,6 +38,10 @@ def get_alpha_c(sam=SAM, cons_dict=CONS_DICT):
     Returns:
         alpha_c (dict): Dictionary of shares of household expenditures
     """
+    if sam is None:
+        sam = read_SAM()
+    if sam is None:
+        raise RuntimeError("SAM data is unavailable. Cannot compute alpha_c.")
     hh_cols = [
         "hhd-r1",
         "hhd-r2",
@@ -55,7 +69,7 @@ def get_alpha_c(sam=SAM, cons_dict=CONS_DICT):
     return alpha_c
 
 
-def get_io_matrix(sam=SAM, cons_dict=CONS_DICT, prod_dict=PROD_DICT):
+def get_io_matrix(sam=None, cons_dict=CONS_DICT, prod_dict=PROD_DICT):
     """
     Calibrate the io_matrix array.  This array relates the share of each
     production category in each consumption category
@@ -68,6 +82,12 @@ def get_io_matrix(sam=SAM, cons_dict=CONS_DICT, prod_dict=PROD_DICT):
     Returns:
         io_df (pd.DataFrame): Dataframe of io_matrix
     """
+    if sam is None:
+        sam = read_SAM()
+    if sam is None:
+        raise RuntimeError(
+            "SAM data is unavailable. Cannot compute io_matrix."
+        )
     # Create initial matrix as dataframe of 0's to fill in
     io_dict = {}
     for key in prod_dict.keys():