Fixes for ixmp4 v0.15.x (#970)

* adjust test_platform fixture to use the new ixmp4 Transport class

* pin ixmp4>=0.15.1

* instantiate ixmp4 settings instead of using old global settings

* adjust imports for renamed facade classes

* Add Max Wolschlager to AUTHORS.rst

* pin ixmp4>=0.15.2

* Update RELEASE_NOTES.md

* fix ixmp4 auth construction

* pin ixmp4>=0.15.3

* dont pass None filters to ixmp4

* pin ixmp4>=0.15.4

* fix ManagerPlatforms init and user getter

* fix expected  exception class

* fix  manager platforms list call

* fix lazy_read_iiasa still using the old settings object

* add tests to cover all of ixmp4.py

* adjust error message regex for InvalidCredentials exception

* fix SceSeAuth class to use scse-toolkit directly and fix the username assertaining logic

* adjust test_platforms.py to assert simpler strings so table rendering can be changed

Author: meksor

AUTHORS.rst

@@ -24,6 +24,7 @@ The following persons contributed to the development of the |pyam| package:
 - Edward Byers `@byersiiasa <https://github.com/byersiiasa>`_
 - Fridolin Glatter `@glatterf42 <https://github.com/glatterf42>`_
 - Linh Ho `@linhho <https://github.com/LinhHo>`
+- Max Wolschlager `@meksor <https://github.com/meksor>`
 
 | The core maintenance of the |pyam| package is done by
   the *Scenario Services & Scientific Software* research theme

RELEASE_NOTES.md

@@ -1,5 +1,9 @@
 # Next Release
 
+## Individual updates
+
+- [#970](https://github.com/IAMconsortium/pyam/pull/970) Fixes for ixmp4 v0.15.x
+
 # Release v3.3.0
 
 ## Highlights
@@ -169,7 +173,7 @@ Credentials to access the IIASA scenario database infrastructure should now be m
 using the **ixmp4** package
 (see [here](https://pyam-iamc.readthedocs.io/en/stable/api/iiasa.html)).
 
-The column *exclude* of the `meta` indicators was moved to a new attribute `exclude`.
+The column _exclude_ of the `meta` indicators was moved to a new attribute `exclude`.
 All validation methods are refactored such that the argument `exclude_on_fail` changes
 this new attribute (see PR [#759](https://github.com/IAMconsortium/pyam/pull/759)).
 
@@ -266,7 +270,7 @@ Bump minimum version of **pandas** to v1.2.0 to support automatic engine selecti
 
 ## Highlights
 
-- Improved performance for writing data to *xlsx*
+- Improved performance for writing data to _xlsx_
 - Support for filtering by model-scenario pairs with an `index` argument
 - Better integration with the IIASA Scenario Explorer database API
 
@@ -367,7 +371,7 @@ pandas [v1.4.0](https://pandas.pydata.org/docs/whatsnew/v1.4.0.html).
 
 ## Highlights
 
-- Update the source code of the manuscript in *Open Research Europe* to reflect changes
+- Update the source code of the manuscript in _Open Research Europe_ to reflect changes
   based on reviewer comments
 - Increase the performance of the IamDataFrame initialization
 - Add an experimental "profiler" module for performance benchmarking

poetry.lock

@@ -6,16 +6,15 @@
 from pathlib import Path
 
 import httpx
-import ixmp4
 import jwt
 import numpy as np
 import pandas as pd
 import requests
 import yaml
 from ixmp4.cli.platforms import tabulate_manager_platforms
-from ixmp4.conf import settings
-from ixmp4.conf.auth import ManagerAuth
+from ixmp4.conf.settings import Settings
 from requests.auth import AuthBase
+from toolkit.client.auth import ManagerAuth, SelfSignedAuth
 
 from pyam.core import IamDataFrame
 from pyam.exceptions import deprecation_warning
@@ -49,16 +48,18 @@ def platforms() -> None:
 
     See Also
     --------
-    ixmp4.conf.settings.manager.list_platforms
+    ixmp4.conf.platforms.ManagerPlatforms.list_platforms
     """
-    tabulate_manager_platforms(ixmp4.conf.settings.manager.list_platforms())
+    settings = Settings()
+    manager_platforms = settings.get_manager_platforms()
+    tabulate_manager_platforms(manager_platforms.list_platforms())
 
 
 def _read_config(file):
     """Read username and password for IIASA API connection from file"""
     with open(file) as stream:
         creds = yaml.safe_load(stream)
-
+    settings = Settings()
     return ManagerAuth(**creds, url=str(settings.manager_url))
 
 
@@ -68,6 +69,8 @@ def _check_response(r, msg="Error connecting to IIASA database", error=RuntimeEr
 
 
 class SceSeAuth(AuthBase):
+    auth: ManagerAuth | SelfSignedAuth | None = None
+
     def __init__(self, creds: str = None, auth_url: str = _AUTH_URL):
         """Connection to the Scenario Services manager service for authentication.
 
@@ -89,7 +92,9 @@ def __init__(self, creds: str = None, auth_url: str = _AUTH_URL):
                 )
                 self.auth = _read_config(DEFAULT_IIASA_CREDS)
             else:
-                self.auth = ixmp4.conf.settings.default_auth
+                settings = Settings()
+                cred_dict = settings.get_credentials().get("default")
+                self.auth = settings.get_client_auth(cred_dict)
         elif isinstance(creds, Path) or is_str(creds):
             deprecation_warning(f"{IXMP4_LOGIN}.", "Using a pyam-credentials file")
             self.auth = _read_config(creds)
@@ -98,19 +103,20 @@ def __init__(self, creds: str = None, auth_url: str = _AUTH_URL):
                 "Passing credentials as clear-text is not allowed. "
                 f"{IXMP4_LOGIN} instead."
             )
-
-        # self.auth is None if connection to manager service cannot be established
-        if self.auth is None:
-            raise httpx.ConnectError("No connection to IIASA manager service.")
-
         # explicit token for anonymous login is not necessary for ixmp4 platforms
         # but is required for legacy Scenario Explorer databases
-        if self.auth.user.username == "@anonymous":
+        if self.auth is None:
             self._get_anonymous_token()
 
         else:
-            self.user = self.auth.user.username
-            self.access_token = self.auth.access_token
+            token_username = getattr(
+                self.auth.access_token.user, "username", "@unknown"
+            )
+            if token_username == "@anonymous":
+                self._get_anonymous_token()
+            else:
+                self.user = token_username
+                self.access_token = self.auth.access_token
 
     def _get_anonymous_token(self):
         r = self.client.get("/legacy/anonym/")
@@ -611,7 +617,9 @@ def read_iiasa(name, default_only=True, meta=True, creds=None, **kwargs):
     Credentials (username & password) are not required to access any public |ixmp4|
     or Scenario Explorer database (i.e., with Guest login).
     """
-    if name in [i.name for i in ixmp4.conf.settings.manager.list_platforms()]:
+    settings = Settings()
+    manager_platforms = settings.get_manager_platforms()
+    if name in [i.name for i in manager_platforms.list_platforms()]:
         if meta is not True:
             raise NotImplementedError(
                 "Reading from ixmp4 platforms requires `meta=True`"
@@ -660,9 +668,9 @@ def lazy_read_iiasa(file, name, default_only=True, meta=True, creds=None, **kwar
     Credentials (username & password) are not required to access any public |ixmp4|
     or Scenario Explorer database (i.e., with Guest login).
     """
-    if name in [
-        platform.name for platform in ixmp4.conf.settings.manager.list_platforms()
-    ]:
+    settings = Settings()
+    manager_platforms = settings.get_manager_platforms()
+    if name in [platform.name for platform in manager_platforms.list_platforms()]:
         raise NotImplementedError(
             "The function `lazy_read_iiasa()` does not support ixmp4 platforms."
         )

pyam/iiasa.py

@@ -2,9 +2,12 @@
 
 import ixmp4
 import pandas as pd
-from ixmp4.core.region import RegionModel
-from ixmp4.core.unit import UnitModel
-from ixmp4.data.abstract import DataPoint
+from ixmp4.core.iamc import DataPoint
+from ixmp4.core.region import Region
+from ixmp4.core.unit import Unit
+from ixmp4.data.iamc.datapoint.filter import FacadeDataPointFilter
+from ixmp4.data.meta.filter import FacadeRunMetaEntryFilter
+from ixmp4.data.run.filter import FacadeRunFilter
 
 logger = logging.getLogger(__name__)
 
@@ -38,11 +41,15 @@ def read_ixmp4(
     if not isinstance(platform, ixmp4.Platform):
         platform = ixmp4.Platform(platform)
 
+    run_filters = FacadeRunFilter(default_only=default_only)
+    if model is not None:
+        run_filters["model"] = model
+    if scenario is not None:
+        run_filters["scenario"] = scenario
+
     # TODO This may have to be revised, see https://github.com/iiasa/ixmp4/issues/72
-    meta_filters = dict(
-        run=dict(default_only=default_only, model=model, scenario=scenario)
-    )
-    iamc_filters = dict(run=dict(default_only=default_only))
+    meta_filters = FacadeRunMetaEntryFilter(run=run_filters)
+    iamc_filters = FacadeDataPointFilter(run=dict(default_only=default_only))
     for key, value in (
         ("model", model),
         ("scenario", scenario),
@@ -97,21 +104,24 @@ def read_run(
     """
     from pyam import IamDataFrame
 
-    if year is not None:
-        raise NotImplementedError("Filter by 'year' not implemented in ixmp4.")
+    iamc_filters = FacadeDataPointFilter()
+    for key, value in (
+        ("region", region),
+        ("variable", variable),
+        ("unit", unit),
+        ("year", year),
+    ):
+        if value is not None:
+            iamc_filters[key] = value
 
-    meta = pd.DataFrame.from_dict(run.meta, orient="index").T
+    meta = pd.DataFrame.from_dict(dict(run.meta), orient="index").T
     meta.index = pd.MultiIndex.from_tuples(
-        [(run.model.name, run.scenario.name)], name=["model", "scenario"]
+        [(run.model.name, run.scenario.name)], names=["model", "scenario"]
     )
     meta["version"] = run.version
 
     return IamDataFrame(
-        data=run.iamc.tabulate(
-            region=region,
-            variable=variable,
-            unit=unit,
-        ),
+        data=run.iamc.tabulate(**iamc_filters),
         meta=meta,
         model=run.model.name,
         scenario=run.scenario.name,
@@ -179,8 +189,8 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df, checkpoint_message: str):
 def _validate_dimensions(platform, df):
     """Ensure that all regions and units in the DataFrame exist in the platform"""
     for dimension, values, model in [
-        ("regions", df.region, RegionModel),
-        ("units", df.unit, UnitModel),
+        ("regions", df.region, Region),
+        ("units", df.unit, Unit),
     ]:
         platform_values = getattr(platform, dimension).tabulate().name.values
         if missing := set(values).difference(platform_values):

pyam/ixmp4.py

@@ -40,7 +40,7 @@ classifiers = [
 # Please also add a section "Dependency changes" to the release notes
 dependencies = [
     "iam-units>=2020.4.21",
-    "ixmp4>=0.13.0",
+    "ixmp4>=0.15.4",
     "matplotlib>=3.6.0",
     "numpy>=1.26.0",
     "openpyxl>=3.1.2",

pyproject.toml

@@ -11,9 +11,9 @@
 import pandas as pd
 import pytest
 from httpx import ConnectError
-from ixmp4.conf.base import PlatformInfo
 from ixmp4.core import Platform
-from ixmp4.data.backend import SqliteTestBackend
+from ixmp4.db.models import get_metadata
+from ixmp4.transport import DirectTransport
 
 from pyam import IamDataFrame, iiasa
 from pyam.utils import IAMC_IDX, META_IDX
@@ -262,19 +262,18 @@ def plot_stackplot_df():
 
 @pytest.fixture(scope="function")
 def test_platform():
-    sqlite = SqliteTestBackend(
-        PlatformInfo(name="sqlite-test", dsn="sqlite:///:memory:")
-    )
-    sqlite.setup()
+    transport = DirectTransport.from_dsn("sqlite:///:memory:")
+    bind = transport.session.bind
+    meta = get_metadata()
+    meta.create_all(bind=bind, tables=meta.sorted_tables, checkfirst=True)
 
-    platform = Platform(_backend=sqlite)
+    platform = Platform(transport)
     platform.regions.create(name="World", hierarchy="common")
     platform.units.create(name="EJ/yr")
 
     yield platform
 
-    sqlite.close()
-    sqlite.teardown()
+    meta.drop_all(bind=bind, checkfirst=True)
 
 
 @pytest.fixture(scope="session")

tests/conftest.py

@@ -7,7 +7,7 @@
 import pandas.testing as pdt
 import pytest
 import yaml
-from ixmp4.core.exceptions import InvalidCredentials
+from toolkit.exceptions import InvalidCredentials
 
 from pyam import IamDataFrame, iiasa, lazy_read_iiasa, read_iiasa
 from pyam.testing import assert_iamframe_equal
@@ -58,10 +58,9 @@
 def test_platforms(capsys):
     # test that the function does not raise an error
     iiasa.platforms()
-    assert (
-        "public-test         public    This is a public ixmp4 test instance"
-        in capsys.readouterr().out
-    )
+    output = capsys.readouterr().out
+    assert "Public Testing Instance" in output
+    assert "This is a public ixmp4 test instance" in output
 
 
 def test_unknown_conn():
@@ -89,7 +88,7 @@ def test_conn_nonexisting_creds_file():
 @pytest.mark.parametrize(
     "creds, error, match",
     [
-        (dict(username="foo", password="bar"), InvalidCredentials, " rejected "),
+        (dict(username="foo", password="bar"), InvalidCredentials, "rejected"),
         (dict(username="user"), TypeError, "missing 1 required .* 'password'"),
     ],
 )

tests/test_iiasa.py

@@ -1,6 +1,9 @@
+from unittest.mock import patch
+
+import ixmp4 as ixmp4_module
 import pytest
-from ixmp4.core.region import RegionModel
-from ixmp4.core.unit import UnitModel
+from ixmp4.core.region import Region
+from ixmp4.core.unit import Unit
 
 import pyam
 from pyam import read_ixmp4
@@ -11,14 +14,14 @@
 def test_to_ixmp4_missing_region_raises(test_platform, test_df_year):
     """Writing to platform raises if region not defined"""
     test_df_year.rename(region={"World": "foo"}, inplace=True)
-    with pytest.raises(RegionModel.NotFound, match="foo. Use `Platform.regions."):
+    with pytest.raises(Region.NotFound, match="foo. Use `Platform.regions."):
         test_df_year.to_ixmp4(platform=test_platform)
 
 
 def test_to_ixmp4_missing_unit_raises(test_platform, test_df_year):
     """Writing to platform raises if unit not defined"""
     test_df_year.rename(unit={"EJ/yr": "foo"}, inplace=True)
-    with pytest.raises(UnitModel.NotFound, match="foo. Use `Platform.units."):
+    with pytest.raises(Unit.NotFound, match="foo. Use `Platform.units."):
         test_df_year.to_ixmp4(platform=test_platform)
 
 
@@ -131,3 +134,50 @@ def test_ixmp4_read_run(test_platform, test_df):
     exp = test_df.filter(scenario="scen_a")
     exp.set_meta(1, "version")  # the IamDataFrame read from ixmp4 includes the version
     pyam.assert_iamframe_equal(exp, obs)
+
+
+def test_ixmp4_read_run_with_filters(test_platform, test_df_year):
+    """Read a run with filters"""
+
+    test_df_year.to_ixmp4(platform=test_platform)
+
+    run = test_platform.runs.get("model_a", "scen_a")
+    obs = read_run(run, region="World", variable="Primary Energy", year=2010)
+
+    exp = test_df_year.filter(
+        scenario="scen_a", region="World", variable="Primary Energy", year=2010
+    )
+    exp.set_meta(1, "version")
+    pyam.assert_iamframe_equal(exp, obs)
+
+
+def test_read_ixmp4_string_platform(test_platform, test_df_year):
+    """read_ixmp4 converts a string platform name to a Platform instance"""
+
+    test_df_year.to_ixmp4(platform=test_platform)
+
+    class FakePlatform:
+        def __new__(cls, name):
+            return test_platform
+
+    with patch.object(ixmp4_module, "Platform", FakePlatform):
+        obs = read_ixmp4(platform="test-platform-name")
+
+    test_df_year.set_meta(1, "version")
+    pyam.assert_iamframe_equal(test_df_year, obs)
+
+
+def test_write_to_ixmp4_string_platform(test_platform, test_df_year):
+    """write_to_ixmp4 converts a string platform name to a Platform instance"""
+
+    class FakePlatform:
+        def __new__(cls, name):
+            return test_platform
+
+    with patch.object(ixmp4_module, "Platform", FakePlatform):
+        test_df_year.to_ixmp4(platform="test-platform-name")
+
+    # verify data was written correctly
+    obs = read_ixmp4(platform=test_platform)
+    test_df_year.set_meta(1, "version")
+    pyam.assert_iamframe_equal(test_df_year, obs)