From 6072a97adc5ff8217f2fe396b38f958bf6579b62 Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 14:05:54 +0100 Subject: [PATCH 1/9] Init CI/CD --- .github/workflows/code-quality.yml | 41 ++++++++++ .github/workflows/publish.yml | 35 ++++++++ .github/workflows/test.yml | 43 ++++++++++ .pre-commit-config.yaml | 67 +++------------- Makefile | 10 ++- pyproject.toml | 80 +++++++++++++++++-- setup.py | 18 ----- src/hpc_funcs/environment/__init__.py | 9 +-- src/hpc_funcs/files.py | 10 +-- src/hpc_funcs/lmod/__init__.py | 28 +++---- .../schedulers/uge/environment/__init__.py | 5 +- .../schedulers/uge/monitoring/__init__.py | 11 +-- .../schedulers/uge/monitoring/follow.py | 22 +++-- src/hpc_funcs/schedulers/uge/qacct.py | 8 +- src/hpc_funcs/schedulers/uge/qstat.py | 6 +- src/hpc_funcs/schedulers/uge/qstat_json.py | 20 ++--- src/hpc_funcs/schedulers/uge/qstat_text.py | 35 ++++---- src/hpc_funcs/schedulers/uge/qstat_xml.py | 46 +++++------ .../schedulers/uge/submission/__init__.py | 36 ++++----- src/hpc_funcs/shell/__init__.py | 17 ++-- tests/conftest.py | 2 +- tests/resources/uge/generate_uge_examples.py | 3 - tests/test_lmod.py | 20 +++-- tests/test_shell.py | 1 - tests/test_status.py | 1 - tests/uge/test_qacct.py | 3 +- tests/uge/test_qstat_json.py | 30 +++---- tests/uge/test_qstat_text.py | 6 +- tests/uge/test_qstat_xml.py | 2 +- tests/uge/test_submitting.py | 1 - 30 files changed, 357 insertions(+), 259 deletions(-) create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml delete mode 100644 setup.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..fa8a1a4 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,41 @@ +name: Code Quality + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + code-quality: + name: Lint & Type Check + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff mypy + + - name: Check formatting with Ruff + run: ruff format --check src/ tests/ + + - name: Lint with Ruff + run: ruff check src/ tests/ + + - name: Type check with mypy + run: mypy src/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..cb396b0 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +jobs: + deploy: + name: Build & Publish + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check distribution + run: twine check dist/* + + - name: Publish to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: twine upload dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..390ea6a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: Test + +on: + push: + branches: + - '**' + pull_request: + branches: [main] + +jobs: + test: + name: Test Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + strategy: + matrix: + python-version: ['3.11', '3.12', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Create environment + run: make env python-version=${{ matrix.python-version }} + + - name: Run tests + run: source ./env/bin/activate && make test python=python + + - name: Build package + run: make build + + - name: Check distribution + run: make test-dist + + - name: Test installation from wheel + run: | + pip install dist/*.whl + python -c "import hpc_funcs; print(hpc_funcs.__version__)" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ef85e8..25da4bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude: ^tests/resources/ @@ -9,7 +9,6 @@ repos: exclude: ^tests/resources/ - id: check-yaml args: ["--unsafe"] - - id: check-added-large-files - id: check-ast - id: check-json - id: debug-statements @@ -20,60 +19,12 @@ repos: - id: check-added-large-files args: ['--maxkb=3000'] - - repo: https://github.com/myint/autoflake - rev: v2.3.1 - hooks: - - id: autoflake - name: Removes unused variables - args: - - --in-place - - --remove-all-unused-imports - - --expand-star-imports - - --ignore-init-module-imports - - - repo: https://github.com/pre-commit/mirrors-isort - rev: v5.10.1 - hooks: - - id: isort - name: Sorts imports - args: [ - # Align isort with black formatting - "--multi-line=3", - "--trailing-comma", - "--force-grid-wrap=0", - "--use-parentheses", - "--line-width=99", - ] - - - repo: https://github.com/psf/black - rev: 24.10.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.6 hooks: - - id: black - name: Fixes formatting - language_version: python3 - args: ["--line-length=99"] - - - - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 - hooks: - - id: flake8 - name: Checks pep8 style - args: [ - "--max-line-length=99", - # Ignore imports in init files - "--per-file-ignores=*/__init__.py:F401,setup.py:E121", - # ignore long comments (E501), as long lines are formatted by black - # ignore Whitespace before ':' (E203) - # ignore Line break occurred before a binary operator (W503) - "--ignore=E501,E203,W503", - ] - - # - repo: https://github.com/pre-commit/mirrors-mypy - # rev: v0.782 - # hooks: - # - id: mypy - # args: [--ignore-missing-imports] + - id: ruff + args: [--fix] + - id: ruff-format - repo: local hooks: @@ -86,13 +37,13 @@ repos: - id: jupyisort name: Sorts ipynb imports - entry: jupytext --pipe-fmt ".py" --pipe "isort - --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=99" --sync + entry: jupytext --pipe-fmt ".py" --pipe "ruff check --select I --fix -" --sync files: \.ipynb$ language: python - - id: jupyblack + - id: jupyformat name: Fixes ipynb format - entry: jupytext --pipe-fmt ".py" --pipe "black - --line-length=99" --sync + entry: jupytext --pipe-fmt ".py" --pipe "ruff format -" --sync files: \.ipynb$ language: python diff --git a/Makefile b/Makefile index b8f7c15..0da135a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: update-format format test test-dist build types upload cov +.PHONY: update-format format check test test-dist build types upload cov clean clean-env package=hpc_funcs version_file1=./src/hpc_funcs/version.py @@ -21,8 +21,8 @@ env: ${env}_uv env_uv: uv venv ${env} - uv pip install -r requirements.txt --python ${env}/bin/python uv pip install -e . --python ${env}/bin/python + uv pip install -e .[dev] --python ${env}/bin/python ${python} -m pre_commit install env_conda: @@ -39,6 +39,8 @@ update-format: format: ${python} -m pre_commit run --all-files +check: format types + test: ${python} -m pytest ./tests @@ -107,3 +109,7 @@ clean: rm -rf *.whl rm -rf ./build/ ./__pycache__/ rm -rf ./dist/ + +clean-env: + rm -rf ./${env}/ + rm -f .git/hooks/pre-commit diff --git a/pyproject.toml b/pyproject.toml index 7325021..91ab9cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,86 @@ [build-system] -requires = ["setuptools", "setuptools-scm"] +requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "hpce_utils" +name = "hpc_funcs" dynamic = ["version"] authors = [] +description = "HPC utility functions" +readme = "README.md" +license = {text = "MIT"} requires-python = ">=3.11" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [] + +[project.optional-dependencies] +dev = [ + "ruff", + "mypy", + "pip", + "pre-commit", + "pytest", + "pytest-cov", + "twine", + "build", + "monkeytype", +] [tool.setuptools.dynamic] -version = {attr = "hpce_utils.version.VERSION"} +version = {attr = "hpc_funcs.version.__version__"} -[options.packages.find] -where="src" +[tool.setuptools.packages.find] +where = ["src"] [tool.setuptools.package-data] -"*" = ["*.jinja"] +"*" = ["*.jinja", "py.typed"] + +[tool.ruff] +line-length = 99 +target-version = "py311" +exclude = [ + ".git", + ".github", + "__pycache__", + "build", + "dist", + "*.egg-info", + "env", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort (import sorting) + "UP", # pyupgrade (modern Python syntax) + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify +] + +[tool.ruff.lint.isort] +known-first-party = ["hpc_funcs"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["B017", "B023", "E501"] +"*/__init__.py" = ["F401"] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +log_cli = true +log_cli_level = "DEBUG" diff --git a/setup.py b/setup.py deleted file mode 100644 index 9857cd0..0000000 --- a/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -from setuptools import setup # type: ignore - - -def main(): - - setup( - name="hpce_utils", - version="0", - python_requires=">=3.6", - install_requires=[], - packages=["hpce_utils"], - package_dir={"": "src"}, - ) - - return - - -main() diff --git a/src/hpc_funcs/environment/__init__.py b/src/hpc_funcs/environment/__init__.py index 9903807..51c58de 100644 --- a/src/hpc_funcs/environment/__init__.py +++ b/src/hpc_funcs/environment/__init__.py @@ -13,8 +13,7 @@ def get_available_cores() -> int: - - n_cores: Optional[int] + n_cores: int | None n_cores = get_threads() @@ -26,7 +25,7 @@ def get_available_cores() -> int: return n_cores -def get_threads() -> Optional[int]: +def get_threads() -> int | None: """Get number of threads from environmental variables""" n_cores = None @@ -65,7 +64,7 @@ def set_threads(n_cores: int) -> None: os.environ[name] = n_cores_ -def get_shm_path() -> Optional[Path]: +def get_shm_path() -> Path | None: """ Get shared memory path for current node. @@ -106,7 +105,7 @@ def is_notebook() -> bool: return False # Probably standard Python interpreter -def get_environment(env_names: List[str]) -> Dict[str, str]: +def get_environment(env_names: list[str]) -> dict[str, str]: """Get environ variables that matter""" environ = dict() diff --git a/src/hpc_funcs/files.py b/src/hpc_funcs/files.py index 4a02cd9..c83b9a4 100644 --- a/src/hpc_funcs/files.py +++ b/src/hpc_funcs/files.py @@ -1,7 +1,7 @@ import tempfile import weakref from pathlib import Path -from typing import Any, Optional, Union +from typing import Any FILENAMES = tempfile._get_candidate_names() # type: ignore @@ -16,9 +16,9 @@ class WorkDir(tempfile.TemporaryDirectory): def __init__( self, - suffix: Optional[str] = None, - prefix: Optional[str] = None, - dir: Optional[Union[str, Path]] = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | Path | None = None, keep: bool = False, ) -> None: self.keep_directory = keep @@ -28,7 +28,7 @@ def __init__( self, super()._cleanup, # type: ignore self.name, - warn_message="Implicitly cleaning up {!r}".format(self), + warn_message=f"Implicitly cleaning up {self!r}", ) def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: diff --git a/src/hpc_funcs/lmod/__init__.py b/src/hpc_funcs/lmod/__init__.py index a7c034a..bd933ca 100644 --- a/src/hpc_funcs/lmod/__init__.py +++ b/src/hpc_funcs/lmod/__init__.py @@ -12,7 +12,7 @@ logger = logging.getLogger("lmod") -@lru_cache() +@lru_cache def get_lmod_executable() -> Path: """Get the LMOD executable path. @@ -40,9 +40,9 @@ def get_lmod_executable() -> Path: def module( command: str, arguments: str, - cmd: Optional[Path] = None, - env: Optional[Dict[str, str]] = None, -) -> Tuple[Dict[str, str], Optional[str]]: + cmd: Path | None = None, + env: dict[str, str] | None = None, +) -> tuple[dict[str, str], str | None]: """Use lmod to execute environmental changes. Args: @@ -108,8 +108,7 @@ def _filter(line: str) -> bool: return True - def _split_line(line: str) -> Tuple[str, str]: - + def _split_line(line: str) -> tuple[str, str]: # format: # os.environ["key"] = "value:value" @@ -140,9 +139,8 @@ def _split_line(line: str) -> Tuple[str, str]: return environment_update, stderr -def update_environment(update_dict: Dict[str, str]) -> None: - - pythonpath = update_dict.get("PYTHONPATH", None) +def update_environment(update_dict: dict[str, str]) -> None: + pythonpath = update_dict.get("PYTHONPATH") os.environ.update(update_dict) @@ -168,13 +166,13 @@ def purge() -> None: module("purge", "") -def load(module_name: str, env: Optional[Dict[str, str]] = None) -> None: +def load(module_name: str, env: dict[str, str] | None = None) -> None: """use `module load` to overload your environment""" update_dict = get_load_environment(module_name, env=env) update_environment(update_dict) -def get_load_environment(module_name: str, env: Optional[Dict[str, str]] = None) -> Dict[str, str]: +def get_load_environment(module_name: str, env: dict[str, str] | None = None) -> dict[str, str]: """use `module load` to overload your environment""" update_dict, _ = module("load", module_name, env=env) return update_dict @@ -186,7 +184,7 @@ def use(path: Path | str) -> None: update_environment(update_dict) -def get_modules() -> Dict[int, str]: +def get_modules() -> dict[int, str]: """Return all active LMOD modules. Hidden modules are ignored. @@ -222,7 +220,6 @@ def _filter(line: str): modules = dict() for line in lines: - # Standardize the line line = " ".join(line.strip().split()) @@ -231,8 +228,7 @@ def _filter(line: str): mods = [x.strip() for x in mods if len(x)] # The delimiters are kept, so select every second - for key, mod in zip(mods[::2], mods[1::2]): - + for key, mod in zip(mods[::2], mods[1::2], strict=False): if "(H)" in mod: continue @@ -243,7 +239,7 @@ def _filter(line: str): return modules -def get_paths() -> List[str]: +def get_paths() -> list[str]: """Return all LMOD paths in use""" paths = os.environ.get("MODULEPATH", "") paths_ = paths.split(":") diff --git a/src/hpc_funcs/schedulers/uge/environment/__init__.py b/src/hpc_funcs/schedulers/uge/environment/__init__.py index a9a8524..a68c501 100644 --- a/src/hpc_funcs/schedulers/uge/environment/__init__.py +++ b/src/hpc_funcs/schedulers/uge/environment/__init__.py @@ -32,7 +32,7 @@ def is_job() -> bool: return True -def get_env() -> Dict[str, Optional[str]]: +def get_env() -> dict[str, str | None]: """ Get all UGE related environmental variables. @@ -69,7 +69,7 @@ def get_tmpdir() -> Path: return path -def get_config() -> Dict[str, Any]: +def get_config() -> dict[str, Any]: """Get UGE configuration - Number of cores available on node @@ -151,7 +151,6 @@ def source(bashfile): variables = dict() for line in lines: - line = line.split("=") # Ignore wrong lines diff --git a/src/hpc_funcs/schedulers/uge/monitoring/__init__.py b/src/hpc_funcs/schedulers/uge/monitoring/__init__.py index ddcf68d..7d7c43b 100644 --- a/src/hpc_funcs/schedulers/uge/monitoring/__init__.py +++ b/src/hpc_funcs/schedulers/uge/monitoring/__init__.py @@ -2,7 +2,8 @@ import logging import time from collections import defaultdict -from typing import Any, Dict, Iterator, List +from collections.abc import Iterator +from typing import Any, Dict, List from hpc_funcs.schedulers.uge.constants import TAGS_RUNNING from hpc_funcs.schedulers.uge.qstat import get_all_jobs_text @@ -12,7 +13,7 @@ logger = logging.getLogger(__name__) -def get_cluster_usage() -> Dict[str, int]: +def get_cluster_usage() -> dict[str, int]: """Get cluster usage information, grouped by users. Returns: @@ -29,7 +30,7 @@ def get_cluster_usage() -> Dict[str, int]: running_jobs = [j for j in jobs if j.get(COLUMN_STATE) in TAGS_RUNNING] # Group by user and sum slots - counts: Dict[str, int] = defaultdict(int) + counts: dict[str, int] = defaultdict(int) for job in running_jobs: user = job.get(COLUMN_USER, "unknown") slots = int(job.get(COLUMN_SLOTS, 0)) @@ -39,7 +40,7 @@ def get_cluster_usage() -> Dict[str, int]: return dict(sorted(counts.items(), key=lambda x: x[1])) -def wait_for_jobs(jobs: List[str], sleep: int = 60) -> Iterator[str]: +def wait_for_jobs(jobs: list[str], sleep: int = 60) -> Iterator[str]: """ """ logger.info(f"Waiting for {len(jobs)} job(s) on UGE...") @@ -67,7 +68,7 @@ def wait_for_jobs(jobs: List[str], sleep: int = 60) -> Iterator[str]: end_time = time.time() diff_time = end_time - start_time - logger.info(f"All jobs finished and took {diff_time / 60 / 60 :.2f}h") + logger.info(f"All jobs finished and took {diff_time / 60 / 60:.2f}h") def is_job_done( diff --git a/src/hpc_funcs/schedulers/uge/monitoring/follow.py b/src/hpc_funcs/schedulers/uge/monitoring/follow.py index 79ebba7..4e3d525 100644 --- a/src/hpc_funcs/schedulers/uge/monitoring/follow.py +++ b/src/hpc_funcs/schedulers/uge/monitoring/follow.py @@ -2,7 +2,7 @@ import logging import time from io import StringIO -from typing import Any, Dict, List, Optional +from typing import Any import tqdm @@ -33,12 +33,10 @@ def get_time_from_ugestr(time_str: str) -> float: class TaskarrayProgress: - @staticmethod def by_jobid( - job_id: str, position: int = 0, file: Optional[StringIO] = None + job_id: str, position: int = 0, file: StringIO | None = None ) -> "TaskarrayProgress": - job_id = str(job_id) job_infos, _ = get_qstat_job_text(job_id) if not job_infos: @@ -49,9 +47,9 @@ def by_jobid( def __init__( self, - job_info: Dict, + job_info: dict, position: int = 0, - file: Optional[StringIO] = None, + file: StringIO | None = None, ) -> None: self.position = position self.file = file @@ -65,9 +63,8 @@ def _read_time(timestamp): return time_start def init_bar(self, job_info: dict, job_status: dict) -> None: - - is_xml = job_info.get("JB_ja_structure", None) is not None - is_json = job_info.get("submission_time", None) is not None + is_xml = job_info.get("JB_ja_structure") is not None + is_json = job_info.get("submission_time") is not None # is_text = not is_xml and not is_json job_id = ( @@ -82,7 +79,9 @@ def init_bar(self, job_info: dict, job_status: dict) -> None: # TODO Move submission_time to constant from format - timestamp: str = job_info.get("submission_time") if is_json else job_info.get("JB_submission_time") # type: ignore + timestamp: str = ( + job_info.get("submission_time") if is_json else job_info.get("JB_submission_time") + ) # type: ignore if timestamp is None: raise ValueError("Could not extract timestamp from job_info") time_start = get_time_from_ugestr(timestamp) if is_json else self._read_time(timestamp) @@ -121,8 +120,7 @@ def init_bar(self, job_info: dict, job_status: dict) -> None: # Reset time self.pbar.last_print_t = self.pbar.start_t = time_start - def update(self, joblist: List[Dict[str, Any]] | None = None) -> None: - + def update(self, joblist: list[dict[str, Any]] | None = None) -> None: if joblist is None: jobs = get_all_jobs_text() joblist = parse_taskarray(jobs) diff --git a/src/hpc_funcs/schedulers/uge/qacct.py b/src/hpc_funcs/schedulers/uge/qacct.py index 50d80c0..4b23cdc 100644 --- a/src/hpc_funcs/schedulers/uge/qacct.py +++ b/src/hpc_funcs/schedulers/uge/qacct.py @@ -1,14 +1,12 @@ import logging import subprocess -from typing import Dict, List logger = logging.getLogger(__name__) COL_SPLIT = 13 -def get_job_accounting(job_id: str) -> List[Dict[str, str]]: - +def get_job_accounting(job_id: str) -> list[dict[str, str]]: cmd = f"qacct -j {job_id}" process = subprocess.run( @@ -29,13 +27,13 @@ def get_job_accounting(job_id: str) -> List[Dict[str, str]]: return data -def parse_qacct(stdout: str) -> List[Dict[str, str]]: +def parse_qacct(stdout: str) -> list[dict[str, str]]: """ Output is column-length based and sections split by "=". Returns list key-value dict per section. """ - output: List[Dict[str, str]] = [dict()] + output: list[dict[str, str]] = [dict()] lines = stdout.split("\n") diff --git a/src/hpc_funcs/schedulers/uge/qstat.py b/src/hpc_funcs/schedulers/uge/qstat.py index 8cb4d1f..84ff118 100644 --- a/src/hpc_funcs/schedulers/uge/qstat.py +++ b/src/hpc_funcs/schedulers/uge/qstat.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, List +from typing import Any from .qstat_json import get_qstat_json from .qstat_text import get_qstat_text @@ -7,14 +7,14 @@ logger = logging.getLogger(__name__) -def get_all_jobs_json() -> List[Dict[str, Any]]: +def get_all_jobs_json() -> list[dict[str, Any]]: """Get all jobs for all users (JSON format).""" all_users = '"*"' jobs = get_qstat_json(users=[all_users]) return jobs -def get_all_jobs_text() -> List[Dict[str, Any]]: +def get_all_jobs_text() -> list[dict[str, Any]]: """Get all jobs for all users (text format).""" all_users = '"*"' jobs = get_qstat_text(users=[all_users]) diff --git a/src/hpc_funcs/schedulers/uge/qstat_json.py b/src/hpc_funcs/schedulers/uge/qstat_json.py index d1c3734..21e57ae 100644 --- a/src/hpc_funcs/schedulers/uge/qstat_json.py +++ b/src/hpc_funcs/schedulers/uge/qstat_json.py @@ -1,7 +1,7 @@ import json import logging import subprocess -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any from hpc_funcs.shell import execute @@ -9,8 +9,8 @@ def get_qstat_job_json( - job_id: Union[str, int], -) -> Tuple[List[Dict[str, Any]], List[str]]: + job_id: str | int, +) -> tuple[list[dict[str, Any]], list[str]]: """Get detailed information for a specific job using qstat -j. This returns comprehensive information about a single job, including @@ -60,10 +60,10 @@ def get_qstat_job_json( def get_qstat_json( - users: Optional[List[str]] = None, - queues: Optional[List[str]] = None, - resource_filter: Optional[str] = None, -) -> List[Dict[str, Any]]: + users: list[str] | None = None, + queues: list[str] | None = None, + resource_filter: str | None = None, +) -> list[dict[str, Any]]: """Get job status information from UGE using qstat -json. Args: @@ -112,7 +112,7 @@ def get_qstat_json( return rows -def parse_jobinfo_json(stdout: str) -> Tuple[List[Dict[str, Any]], List[str]]: +def parse_jobinfo_json(stdout: str) -> tuple[list[dict[str, Any]], list[str]]: """Parse job info JSON output. Args: @@ -154,7 +154,7 @@ def parse_jobinfo_json(stdout: str) -> Tuple[List[Dict[str, Any]], List[str]]: return rows, errors -def parse_joblist_json(stdout: str) -> List[Dict[str, str]]: +def parse_joblist_json(stdout: str) -> list[dict[str, str]]: """Parse qstat JSON output into a list of job dictionaries. Args: @@ -193,7 +193,7 @@ def parse_joblist_json(stdout: str) -> List[Dict[str, str]]: return rows -def _extract_job_row(job: Dict[str, Any], job_type: str) -> Dict[str, Any]: +def _extract_job_row(job: dict[str, Any], job_type: str) -> dict[str, Any]: """Extract relevant fields from a job dictionary. Args: diff --git a/src/hpc_funcs/schedulers/uge/qstat_text.py b/src/hpc_funcs/schedulers/uge/qstat_text.py index 7078e84..effed6a 100644 --- a/src/hpc_funcs/schedulers/uge/qstat_text.py +++ b/src/hpc_funcs/schedulers/uge/qstat_text.py @@ -1,7 +1,7 @@ import logging import re import subprocess -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any from .constants import TAGS_ERROR, TAGS_PENDING, TAGS_RUNNING @@ -47,10 +47,10 @@ def get_qstat_text( - users: Optional[List[str]] = None, - queues: Optional[List[str]] = None, - resource_filter: Optional[str] = None, -) -> List[Dict[str, Any]]: + users: list[str] | None = None, + queues: list[str] | None = None, + resource_filter: str | None = None, +) -> list[dict[str, Any]]: """Get job status information from UGE using qstat text output. Args: @@ -105,8 +105,8 @@ def get_qstat_text( def get_qstat_job_text( - job_id: Union[str, int], -) -> Tuple[List[Dict[str, Any]], List[str]]: + job_id: str | int, +) -> tuple[list[dict[str, Any]], list[str]]: """Get detailed information for a specific job using qstat -j (text format). This returns comprehensive information about a single job, including @@ -166,7 +166,7 @@ def get_qstat_job_text( return jobs, errors -def parse_joblist_text(stdout: str) -> List[Dict[str, Any]]: +def parse_joblist_text(stdout: str) -> list[dict[str, Any]]: """ Parse UGE qstat text output (list format). @@ -176,7 +176,7 @@ def parse_joblist_text(stdout: str) -> List[Dict[str, Any]]: Returns: List of dictionaries containing job information """ - jobs: List[Dict[str, Any]] = [] + jobs: list[dict[str, Any]] = [] lines = stdout.strip().split("\n") if len(lines) < 3: @@ -207,10 +207,9 @@ def parse_joblist_text(stdout: str) -> List[Dict[str, Any]]: # Extract fields based on column positions # We'll use the positions to slice the line - job: Dict[str, Any] = {} + job: dict[str, Any] = {} for i, col in enumerate(ordered_cols): - start = column_positions[col] end = column_positions[ordered_cols[i + 1]] if i + 1 < len(ordered_cols) else None value = line[start:end].strip() @@ -222,7 +221,7 @@ def parse_joblist_text(stdout: str) -> List[Dict[str, Any]]: return jobs -def parse_jobinfo_text(stdout: str) -> List[Dict[str, str]]: +def parse_jobinfo_text(stdout: str) -> list[dict[str, str]]: """ Output is column-length based and sections split by "=". Returns list key-value dict per section. @@ -230,7 +229,7 @@ def parse_jobinfo_text(stdout: str) -> List[Dict[str, str]]: COL_VALUE_START = 28 - output: List[Dict[str, str]] = [dict()] + output: list[dict[str, str]] = [dict()] lines = stdout.split("\n") @@ -258,7 +257,7 @@ def parse_jobinfo_text(stdout: str) -> List[Dict[str, str]]: return output -def parse_qstat_text(stdout: str) -> List[Dict[str, Any]]: +def parse_qstat_text(stdout: str) -> list[dict[str, Any]]: """Parse qstat text output into list of job dicts.""" stdout = stdout.strip() lines = stdout.split("\n") @@ -267,7 +266,7 @@ def parse_qstat_text(stdout: str) -> List[Dict[str, Any]]: header.remove("at") header_indicies = [] - rows: List[Dict[str, Any]] = [] + rows: list[dict[str, Any]] = [] for line in header[1:]: idx = lines[0].index(line) @@ -289,7 +288,7 @@ def split_qstat_line(line): line_ = split_qstat_line(line) line_ = list(line_) - row: Dict[str, Any] = {key: value for key, value in zip(header, line_)} + row: dict[str, Any] = {key: value for key, value in zip(header, line_, strict=False)} # Convert slots to int if "slots" in row: row["slots"] = int(row["slots"]) @@ -298,7 +297,7 @@ def split_qstat_line(line): return rows -def parse_taskarray(jobs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +def parse_taskarray(jobs: list[dict[str, Any]]) -> list[dict[str, Any]]: """Parse task array information from job list. Args: @@ -326,7 +325,7 @@ def _parse_task_count(line: str) -> int: # Get unique job IDs job_ids = set(job[COLUMN_JOBID] for job in jobs) - rows: List[Dict[str, Any]] = [] + rows: list[dict[str, Any]] = [] for job_id in job_ids: # Filter jobs by job_id diff --git a/src/hpc_funcs/schedulers/uge/qstat_xml.py b/src/hpc_funcs/schedulers/uge/qstat_xml.py index 1fe04fe..04f1ae5 100644 --- a/src/hpc_funcs/schedulers/uge/qstat_xml.py +++ b/src/hpc_funcs/schedulers/uge/qstat_xml.py @@ -1,15 +1,15 @@ import logging import subprocess import xml.etree.ElementTree as ET -from typing import Any, Dict, List, Union +from typing import Any from xml.etree.ElementTree import Element logger = logging.getLogger(__name__) def get_qstat_job_xml( - job_id: Union[str, int], -) -> List[Dict[str, Any]]: + job_id: str | int, +) -> list[dict[str, Any]]: """Get detailed information for a specific job using qstat -j -xml. This returns comprehensive information about a single job, including @@ -61,16 +61,14 @@ def get_qstat_job_xml( def parse_jobinfo_xml( stdout_xml: str, -) -> List[ - Dict[ +) -> list[ + dict[ str, - Union[ - str, - List[Dict[str, Union[List[Dict[str, str]], str]]], - List[Dict[str, str]], - Dict[str, Dict[str, str]], - Dict[str, str], - ], + str + | list[dict[str, list[dict[str, str]] | str]] + | list[dict[str, str]] + | dict[str, dict[str, str]] + | dict[str, str], ] ]: """ @@ -135,33 +133,29 @@ def parse_element(elem: Element) -> Any: def element_to_dict( elem: Element, -) -> Dict[ +) -> dict[ str, - Union[ - str, - List[Dict[str, Union[List[Dict[str, str]], str]]], - List[Dict[str, str]], - Dict[str, Dict[str, str]], - Dict[str, str], - ], + str + | list[dict[str, list[dict[str, str]] | str]] + | list[dict[str, str]] + | dict[str, dict[str, str]] + | dict[str, str], ]: - children = list(elem) - child_map: Dict[str, List[Any]] = {} + child_map: dict[str, list[Any]] = {} for child in children: child_val = parse_element(child) child_map.setdefault(child.tag, []).append(child_val) - d: Dict[str, Any] = {} + d: dict[str, Any] = {} for tag, items in child_map.items(): d[tag] = items[0] if len(items) == 1 else items return d -def element_to_list(elem: Element) -> List[Dict[str, Union[List[Dict[str, str]], str]]]: - - out: List[Any] = [] +def element_to_list(elem: Element) -> list[dict[str, list[dict[str, str]] | str]]: + out: list[Any] = [] for child in elem: if child.tag != "element": continue diff --git a/src/hpc_funcs/schedulers/uge/submission/__init__.py b/src/hpc_funcs/schedulers/uge/submission/__init__.py index 0aa1ed1..2dba2ef 100644 --- a/src/hpc_funcs/schedulers/uge/submission/__init__.py +++ b/src/hpc_funcs/schedulers/uge/submission/__init__.py @@ -21,15 +21,15 @@ def generate_single_script( cmd: str, cores: int = 1, - cwd: Optional[Path] = None, - environ: Dict[str, str] | None = None, + cwd: Path | None = None, + environ: dict[str, str] | None = None, hours: int = 7, - mins: Optional[int] = None, - log_dir: Optional[Path] = DEFAULT_LOG_DIR, + mins: int | None = None, + log_dir: Path | None = DEFAULT_LOG_DIR, mem: int = 4, name: str = "UGEJob", - hold_job_id: Optional[str] = None, - user_email: Optional[str] = None, + hold_job_id: str | None = None, + user_email: str | None = None, generate_dirs: bool = True, ) -> str: """ @@ -62,19 +62,19 @@ def generate_single_script( def generate_taskarray_script( cmd: str, cores: int = 1, - cwd: Optional[Path] = None, - environ: Dict[str, str] | None = None, + cwd: Path | None = None, + environ: dict[str, str] | None = None, hours: int = 7, - mins: Optional[int] = None, - log_dir: Optional[Path] = DEFAULT_LOG_DIR, + mins: int | None = None, + log_dir: Path | None = DEFAULT_LOG_DIR, mem: int = 4, name: str = "UGEJob", task_concurrent: int = 100, task_start: int = 1, task_step: int = 1, - task_stop: Optional[int] = None, - hold_job_id: Optional[str] = None, - user_email: Optional[str] = None, + task_stop: int | None = None, + hold_job_id: str | None = None, + user_email: str | None = None, generate_dirs: bool = True, ) -> str: """ @@ -114,7 +114,6 @@ def generate_hold_script( log_dir: Path | None = DEFAULT_LOG_DIR, generate_dirs: bool = True, ) -> str: - if generate_dirs: log_dir_str = generate_log_dir(log_dir) else: @@ -135,7 +134,6 @@ def generate_hold_script( def generate_log_dir(log_dir: Path | None) -> str | None: - if log_dir is not None: if not log_dir.exists(): log_dir.mkdir(parents=True) @@ -154,7 +152,7 @@ def read_logfiles( job_id: str, ignore_stdout: bool = True, filter_lmod: bool = False, -) -> Tuple[Dict[Path, List[str]], Dict[Path, List[str]]]: +) -> tuple[dict[Path, list[str]], dict[Path, list[str]]]: """Read logfiles produced by UGE task array. Ignore empty log files""" logger.debug(f"Looking for finished log files in {log_path}") stderr_log_filenames = list(log_path.glob(f"*.e{job_id}*")) @@ -181,7 +179,7 @@ def read_logfiles( return stdout, stderr -def filter_stderr_for_lmod(stderr_dict: Dict[Path, List[str]]) -> Dict[Path, List[str]]: +def filter_stderr_for_lmod(stderr_dict: dict[Path, list[str]]) -> dict[Path, list[str]]: """Filter stderr for lmod lines""" stderr_filtered = defaultdict(list) @@ -194,9 +192,9 @@ def filter_stderr_for_lmod(stderr_dict: Dict[Path, List[str]]) -> Dict[Path, Lis return dict(stderr_filtered) -def parse_logfile(filename: Path) -> List[str]: +def parse_logfile(filename: Path) -> list[str]: """Read logfile, without line-breaks""" # TODO Maybe find exceptions and raise them? - with open(filename, "r") as f: + with open(filename) as f: lines = f.read().split("\n") return lines diff --git a/src/hpc_funcs/shell/__init__.py b/src/hpc_funcs/shell/__init__.py index b5abb17..b921d85 100644 --- a/src/hpc_funcs/shell/__init__.py +++ b/src/hpc_funcs/shell/__init__.py @@ -3,13 +3,14 @@ import shutil import subprocess import time +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, Optional, Tuple, Union +from typing import Optional, Tuple, Union logger = logging.getLogger(__name__) -def which(cmd: Union[Path, str]) -> Optional[Path]: +def which(cmd: Path | str) -> Path | None: """Check if command exists in environment""" path_ = shutil.which(cmd) @@ -21,7 +22,7 @@ def which(cmd: Union[Path, str]) -> Optional[Path]: return path -def switch_workdir(path: Optional[Path]) -> bool: +def switch_workdir(path: Path | None) -> bool: """Check if it makes sense to change directory""" if path is None: @@ -97,7 +98,7 @@ def close(self) -> None: self._process.terminate() -def stream(cmd: str, cwd: Optional[Path] = None, shell: bool = True) -> StreamResult: +def stream(cmd: str, cwd: Path | None = None, shell: bool = True) -> StreamResult: """Execute command in directory, and stream stdout. Returns a StreamResult object that can be iterated to get stdout lines. @@ -133,11 +134,11 @@ def stream(cmd: str, cwd: Optional[Path] = None, shell: bool = True) -> StreamRe def execute( cmd: str, - cwd: Optional[Path] = None, + cwd: Path | None = None, shell: bool = True, timeout: None = None, check: bool = True, -) -> Tuple[str, str]: +) -> tuple[str, str]: """Execute command in directory, and return stdout and stderr :param cmd: The shell command @@ -192,12 +193,12 @@ def execute( def execute_with_retry( cmd: str, - cwd: Optional[Path] = None, + cwd: Path | None = None, shell: bool = True, timeout: None = None, max_retries: int = 3, update_interval: int = 5, -) -> Tuple[str, str]: +) -> tuple[str, str]: """Execute command in directory, and return stdout and stderr :param cmd: The shell command diff --git a/tests/conftest.py b/tests/conftest.py index 349c206..6a11e2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os import tempfile +from collections.abc import Generator from pathlib import Path -from typing import Generator import pytest diff --git a/tests/resources/uge/generate_uge_examples.py b/tests/resources/uge/generate_uge_examples.py index bd4c899..998b655 100644 --- a/tests/resources/uge/generate_uge_examples.py +++ b/tests/resources/uge/generate_uge_examples.py @@ -10,7 +10,6 @@ def generate_taskarray_log(global_tmp_path: Path): - # Need network accessible folder tmp_path = global_tmp_path @@ -46,7 +45,6 @@ def generate_taskarray_log(global_tmp_path: Path): def generate_errorjob_log(global_tmp_path: Path): - # Need network accessible folder tmp_path = global_tmp_path @@ -84,7 +82,6 @@ def generate_joblists(): if __name__ == "__main__": - tmp_path = Path("./examples") tmp_path.mkdir() diff --git a/tests/test_lmod.py b/tests/test_lmod.py index 0a483c4..e9567b0 100644 --- a/tests/test_lmod.py +++ b/tests/test_lmod.py @@ -14,16 +14,15 @@ def test_use() -> None: - lmod.use(MODULE_PATH) paths = os.environ.get("MODULEPATH") print(paths) assert os.environ.get("MODULEPATH") is not None, "No module path to be found" - assert str(MODULE_PATH) in os.environ.get( - "MODULEPATH", "" - ), "Unable to find loaded module path" + assert str(MODULE_PATH) in os.environ.get("MODULEPATH", ""), ( + "Unable to find loaded module path" + ) def test_load_os() -> None: @@ -32,9 +31,9 @@ def test_load_os() -> None: print(os.environ.get("MODULEPATH")) # Check use - assert str(MODULE_PATH) in os.environ.get( - "MODULEPATH", "" - ), "Unable to find loaded module path" + assert str(MODULE_PATH) in os.environ.get("MODULEPATH", ""), ( + "Unable to find loaded module path" + ) lmod.load(MODULE_NAME) @@ -54,15 +53,14 @@ def test_load_os() -> None: def test_load_return() -> None: - lmod.use(MODULE_PATH) print(os.environ.get("MODULEPATH")) # Check use - assert str(MODULE_PATH) in os.environ.get( - "MODULEPATH", "" - ), "Unable to find loaded module path" + assert str(MODULE_PATH) in os.environ.get("MODULEPATH", ""), ( + "Unable to find loaded module path" + ) update_dict = lmod.get_load_environment(MODULE_NAME) diff --git a/tests/test_shell.py b/tests/test_shell.py index 07c086e..d6c52bb 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -1,7 +1,6 @@ import subprocess import pytest - from hpce_utils import shell diff --git a/tests/test_status.py b/tests/test_status.py index a28804d..d9d8111 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -4,7 +4,6 @@ from unittest.mock import MagicMock, patch import pytest - from hpce_utils.managers.uge import status, submitting pytest.skip(allow_module_level=True) diff --git a/tests/uge/test_qacct.py b/tests/uge/test_qacct.py index dc998dd..796e066 100644 --- a/tests/uge/test_qacct.py +++ b/tests/uge/test_qacct.py @@ -5,10 +5,9 @@ def test_parse_text(): - filename = RESOURCES / "uge" / "qacct_array.txt" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() data = qacct.parse_qacct(stdout) diff --git a/tests/uge/test_qstat_json.py b/tests/uge/test_qstat_json.py index 8e9589a..435296e 100644 --- a/tests/uge/test_qstat_json.py +++ b/tests/uge/test_qstat_json.py @@ -8,7 +8,7 @@ def test_parse_joblist_json(): filename = RESOURCES / "uge" / "qstat_joblist.json" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() rows = parse_joblist_json(stdout) @@ -42,14 +42,14 @@ def test_parse_joblist_json(): # Verify first job has sanitized data first_job = rows[0] - assert first_job["owner"].startswith( - "user" - ), f"Expected sanitized username, got: {first_job['owner']}" + assert first_job["owner"].startswith("user"), ( + f"Expected sanitized username, got: {first_job['owner']}" + ) # Check queue names are sanitized - assert ( - "example.com" in first_job["queue_name"] - ), f"Expected sanitized hostname in queue, got: {first_job['queue_name']}" + assert "example.com" in first_job["queue_name"], ( + f"Expected sanitized hostname in queue, got: {first_job['queue_name']}" + ) def test_parse_jobinfo_json(): @@ -57,7 +57,7 @@ def test_parse_jobinfo_json(): filename = RESOURCES / "uge" / "qstat_jobinfo_array.json" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() rows, errors = parse_jobinfo_json(stdout) @@ -93,22 +93,22 @@ def test_parse_jobinfo_error_json(): filename = RESOURCES / "uge" / "qstat_jobinfo_error.json" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() rows, errors = parse_jobinfo_json(stdout) # Verify error lines were extracted assert len(errors) > 0, "Expected error lines to be extracted" - assert any( - "error reason" in err for err in errors - ), f"Expected 'error reason' in errors, got: {errors}" + assert any("error reason" in err for err in errors), ( + f"Expected 'error reason' in errors, got: {errors}" + ) # Verify the error contains the expected information first_error = errors[0] - assert ( - "Permission denied" in first_error - ), f"Expected 'Permission denied' in error, got: {first_error}" + assert "Permission denied" in first_error, ( + f"Expected 'Permission denied' in error, got: {first_error}" + ) # Verify structure assert len(rows) > 0, "No jobs returned" diff --git a/tests/uge/test_qstat_text.py b/tests/uge/test_qstat_text.py index 2db667f..90e2d7d 100644 --- a/tests/uge/test_qstat_text.py +++ b/tests/uge/test_qstat_text.py @@ -15,10 +15,9 @@ def test_parse_jobinfo(): - filename = RESOURCES / "uge" / "qstat_jobinfo_array.txt" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() jobs = parse_jobinfo_text(stdout) @@ -40,10 +39,9 @@ def test_parse_jobinfo(): def test_parse_joblist(): - filename = RESOURCES / "uge/qstat_joblist.txt" - with open(filename, "r") as f: + with open(filename) as f: stdout = f.read() job_list = parse_joblist_text(stdout) diff --git a/tests/uge/test_qstat_xml.py b/tests/uge/test_qstat_xml.py index d681c86..0477159 100644 --- a/tests/uge/test_qstat_xml.py +++ b/tests/uge/test_qstat_xml.py @@ -8,7 +8,7 @@ def test_parse_jobinfo_xml(): filename = RESOURCES / "uge" / "qstat_jobinfo_array.xml" - with open(filename, "r") as f: + with open(filename) as f: xml_str = f.read() jobs = parse_jobinfo_xml(xml_str) diff --git a/tests/uge/test_submitting.py b/tests/uge/test_submitting.py index fe78d17..dbc8440 100644 --- a/tests/uge/test_submitting.py +++ b/tests/uge/test_submitting.py @@ -131,7 +131,6 @@ def test_taskarray(global_tmp_path: Path): # Parse output for _, lines in stdout.items(): - for line in lines: if not line.strip(): continue From 298c98f3bccb54cc66f2c8ddb977f03da69cf78f Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 14:36:29 +0100 Subject: [PATCH 2/9] Replaced mypy with ty type checker, added JSON test exclusions, fixed type hints and code style. --- .pre-commit-config.yaml | 7 +++--- pyproject.toml | 19 +++++++++++----- src/hpc_funcs/environment/__init__.py | 4 ++-- src/hpc_funcs/files.py | 11 +++++++--- src/hpc_funcs/lmod/__init__.py | 14 ++++-------- .../schedulers/uge/environment/__init__.py | 19 +++++----------- .../schedulers/uge/monitoring/__init__.py | 7 ++---- .../schedulers/uge/monitoring/follow.py | 2 +- src/hpc_funcs/schedulers/uge/qacct.py | 4 ++-- src/hpc_funcs/schedulers/uge/qstat_text.py | 8 +++---- src/hpc_funcs/schedulers/uge/qsub.py | 9 +++----- .../schedulers/uge/submission/__init__.py | 8 +++---- src/hpc_funcs/shell/__init__.py | 8 +++---- tests/test_status.py | 22 ++++++++++--------- tests/uge/test_monitoring.py | 1 - tests/uge/test_qstat.py | 2 -- 16 files changed, 68 insertions(+), 77 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25da4bb..66ceea6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: args: ["--unsafe"] - id: check-ast - id: check-json + exclude: ^tests/resources/uge/qstat_jobinfo_(array|error)\.json$ - id: debug-statements - id: detect-aws-credentials args: [--allow-missing-credentials] @@ -29,10 +30,10 @@ repos: - repo: local hooks: - - id: mypy + - id: ty name: Static type checking - entry: mypy - files: \.py$ + entry: ty check + files: ^src/.*\.py$ language: python - id: jupyisort diff --git a/pyproject.toml b/pyproject.toml index 91ab9cc..3832425 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [] [project.optional-dependencies] dev = [ "ruff", - "mypy", + "ty", "pip", "pre-commit", "pytest", @@ -74,11 +74,18 @@ known-first-party = ["hpc_funcs"] "tests/*" = ["B017", "B023", "E501"] "*/__init__.py" = ["F401"] -[tool.mypy] -python_version = "3.11" -warn_return_any = true -warn_unused_configs = true -ignore_missing_imports = true +[tool.ty] +# Type checking with ty (Astral's extremely fast Python type checker) + +[tool.ty.environment] +python-version = "3.11" + +[tool.ty.src] +exclude = ["tests/"] + +[tool.ty.rules] +# Ignore unresolved imports since pre-commit env doesn't have all packages +unresolved-import = "ignore" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/hpc_funcs/environment/__init__.py b/src/hpc_funcs/environment/__init__.py index 51c58de..76940e8 100644 --- a/src/hpc_funcs/environment/__init__.py +++ b/src/hpc_funcs/environment/__init__.py @@ -1,7 +1,7 @@ import multiprocessing import os from pathlib import Path -from typing import Dict, List, Optional +from typing import Optional ENVIRON_CORES = [ "OMP_NUM_THREADS", @@ -107,7 +107,7 @@ def is_notebook() -> bool: def get_environment(env_names: list[str]) -> dict[str, str]: """Get environ variables that matter""" - environ = dict() + environ = {} for var in env_names: value = os.environ.get(var, None) diff --git a/src/hpc_funcs/files.py b/src/hpc_funcs/files.py index c83b9a4..211ce49 100644 --- a/src/hpc_funcs/files.py +++ b/src/hpc_funcs/files.py @@ -1,7 +1,7 @@ import tempfile import weakref from pathlib import Path -from typing import Any +from types import TracebackType FILENAMES = tempfile._get_candidate_names() # type: ignore @@ -31,9 +31,14 @@ def __init__( warn_message=f"Implicitly cleaning up {self!r}", ) - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__( + self, + exc: type[BaseException] | None, + value: BaseException | None, + tb: TracebackType | None, + ) -> None: if not self.keep_directory: - super().__exit__(exc_type, exc_val, exc_tb) + super().__exit__(exc, value, tb) def get_path(self) -> Path: return Path(self.name) diff --git a/src/hpc_funcs/lmod/__init__.py b/src/hpc_funcs/lmod/__init__.py index bd933ca..83069a9 100644 --- a/src/hpc_funcs/lmod/__init__.py +++ b/src/hpc_funcs/lmod/__init__.py @@ -5,7 +5,7 @@ import sys from functools import lru_cache from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from hpc_funcs.shell import which @@ -103,10 +103,7 @@ def _filter(line: str) -> bool: if "_LMFILES_" in line: return False - if "_ModuleTable" in line: - return False - - return True + return "_ModuleTable" not in line def _split_line(line: str) -> tuple[str, str]: # format: @@ -210,15 +207,12 @@ def _filter(line: str): if line[0] != " ": return False - if ")" not in line: - return False - - return True + return ")" in line # Filter to only lines with modules lines = [line for line in lines if _filter(line)] - modules = dict() + modules = {} for line in lines: # Standardize the line line = " ".join(line.strip().split()) diff --git a/src/hpc_funcs/schedulers/uge/environment/__init__.py b/src/hpc_funcs/schedulers/uge/environment/__init__.py index a68c501..b04338a 100644 --- a/src/hpc_funcs/schedulers/uge/environment/__init__.py +++ b/src/hpc_funcs/schedulers/uge/environment/__init__.py @@ -1,7 +1,7 @@ import os import shutil from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Optional from hpc_funcs.shell import execute @@ -15,10 +15,7 @@ def has_uge() -> bool: cmd = shutil.which(COMMAND_SUBMIT) - if cmd is not None: - return True - - return False + return cmd is not None def is_job() -> bool: @@ -26,10 +23,7 @@ def is_job() -> bool: name = os.getenv("SGE_TASK_ID") - if name is None: - return False - - return True + return name is not None def get_env() -> dict[str, str | None]: @@ -127,10 +121,7 @@ def is_interactive(): return False # if request is qrlogin, then qrsh was used - if uge_type == "QRLOGIN": - return True - - return False + return uge_type == "QRLOGIN" def source(bashfile): @@ -148,7 +139,7 @@ def source(bashfile): stdout, _ = execute(cmd) lines = stdout.split("\n") - variables = dict() + variables = {} for line in lines: line = line.split("=") diff --git a/src/hpc_funcs/schedulers/uge/monitoring/__init__.py b/src/hpc_funcs/schedulers/uge/monitoring/__init__.py index 7d7c43b..9a9bb4a 100644 --- a/src/hpc_funcs/schedulers/uge/monitoring/__init__.py +++ b/src/hpc_funcs/schedulers/uge/monitoring/__init__.py @@ -3,7 +3,7 @@ import time from collections import defaultdict from collections.abc import Iterator -from typing import Any, Dict, List +from typing import Any from hpc_funcs.schedulers.uge.constants import TAGS_RUNNING from hpc_funcs.schedulers.uge.qstat import get_all_jobs_text @@ -82,9 +82,6 @@ def is_job_done( job_info, _ = get_qstat_job_json(job_id) # If there still is some qstat information, the job is not done - if job_info: - return False - # TODO Check qacct -j information - return True + return not job_info diff --git a/src/hpc_funcs/schedulers/uge/monitoring/follow.py b/src/hpc_funcs/schedulers/uge/monitoring/follow.py index 4e3d525..a37b8a4 100644 --- a/src/hpc_funcs/schedulers/uge/monitoring/follow.py +++ b/src/hpc_funcs/schedulers/uge/monitoring/follow.py @@ -145,7 +145,7 @@ def update(self, joblist: list[dict[str, Any]] | None = None) -> None: n_error = int(job.get(COLUMN_ERROR, 0)) n_finished = self.n_total - n_pending - n_running - postfix = dict() + postfix = {} if n_error > 0: postfix["err"] = n_error diff --git a/src/hpc_funcs/schedulers/uge/qacct.py b/src/hpc_funcs/schedulers/uge/qacct.py index 4b23cdc..2a5683f 100644 --- a/src/hpc_funcs/schedulers/uge/qacct.py +++ b/src/hpc_funcs/schedulers/uge/qacct.py @@ -33,14 +33,14 @@ def parse_qacct(stdout: str) -> list[dict[str, str]]: Returns list key-value dict per section. """ - output: list[dict[str, str]] = [dict()] + output: list[dict[str, str]] = [{}] lines = stdout.split("\n") for line in lines: if "===========" in line: if len(output[-1]) > 1: - output += [dict()] + output += [{}] continue # Format: pe_taskid NONE diff --git a/src/hpc_funcs/schedulers/uge/qstat_text.py b/src/hpc_funcs/schedulers/uge/qstat_text.py index effed6a..a44d3ef 100644 --- a/src/hpc_funcs/schedulers/uge/qstat_text.py +++ b/src/hpc_funcs/schedulers/uge/qstat_text.py @@ -229,14 +229,14 @@ def parse_jobinfo_text(stdout: str) -> list[dict[str, str]]: COL_VALUE_START = 28 - output: list[dict[str, str]] = [dict()] + output: list[dict[str, str]] = [{}] lines = stdout.split("\n") for line in lines: if "=" * 5 in line: if len(output[-1]) > 1: - output += [dict()] + output += [{}] continue # Format: pe_taskid NONE @@ -288,7 +288,7 @@ def split_qstat_line(line): line_ = split_qstat_line(line) line_ = list(line_) - row: dict[str, Any] = {key: value for key, value in zip(header, line_, strict=False)} + row: dict[str, Any] = dict(zip(header, line_, strict=False)) # Convert slots to int if "slots" in row: row["slots"] = int(row["slots"]) @@ -323,7 +323,7 @@ def _parse_task_count(line: str) -> int: return count # Get unique job IDs - job_ids = set(job[COLUMN_JOBID] for job in jobs) + job_ids = {job[COLUMN_JOBID] for job in jobs} rows: list[dict[str, Any]] = [] diff --git a/src/hpc_funcs/schedulers/uge/qsub.py b/src/hpc_funcs/schedulers/uge/qsub.py index 0036b74..b2aa9c4 100644 --- a/src/hpc_funcs/schedulers/uge/qsub.py +++ b/src/hpc_funcs/schedulers/uge/qsub.py @@ -27,10 +27,7 @@ def write_script( Raises: ValueError: If directory exists but is not a directory. """ - if directory is None: - directory = Path("./") - else: - directory = Path(directory) + directory = Path("./") if directory is None else Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -103,8 +100,8 @@ def submit_script(script_path: Path | str) -> str: # Validate format of job_id try: int(uge_id) - except ValueError: - raise RuntimeError(f"UGE Job ID is not a valid number: '{uge_id}'") + except ValueError as err: + raise RuntimeError(f"UGE Job ID is not a valid number: '{uge_id}'") from err logger.info(f"Submitted job: {uge_id}") diff --git a/src/hpc_funcs/schedulers/uge/submission/__init__.py b/src/hpc_funcs/schedulers/uge/submission/__init__.py index 2dba2ef..16f4ecb 100644 --- a/src/hpc_funcs/schedulers/uge/submission/__init__.py +++ b/src/hpc_funcs/schedulers/uge/submission/__init__.py @@ -1,7 +1,7 @@ import logging from collections import defaultdict from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Optional from jinja2 import Template @@ -157,7 +157,7 @@ def read_logfiles( logger.debug(f"Looking for finished log files in {log_path}") stderr_log_filenames = list(log_path.glob(f"*.e{job_id}*")) - stderr = dict() + stderr = {} for filename in stderr_log_filenames: if filename.stat().st_size == 0: continue @@ -167,10 +167,10 @@ def read_logfiles( stderr = filter_stderr_for_lmod(stderr) if ignore_stdout: - return dict(), stderr + return {}, stderr stdout_log_filenames = log_path.glob(f"*.o{job_id}*") - stdout = dict() + stdout = {} for filename in stdout_log_filenames: if filename.stat().st_size == 0: continue diff --git a/src/hpc_funcs/shell/__init__.py b/src/hpc_funcs/shell/__init__.py index b921d85..e53a89a 100644 --- a/src/hpc_funcs/shell/__init__.py +++ b/src/hpc_funcs/shell/__init__.py @@ -5,7 +5,7 @@ import time from collections.abc import Iterator from pathlib import Path -from typing import Optional, Tuple, Union +from typing import Optional logger = logging.getLogger(__name__) @@ -66,8 +66,7 @@ def __iter__(self) -> Iterator[str]: if self._process.stdout is None: return - for line in iter(self._process.stdout.readline, ""): - yield line + yield from iter(self._process.stdout.readline, "") # Capture stderr after stdout is exhausted if self._process.stderr is not None: @@ -209,7 +208,8 @@ def execute_with_retry( :param update_interval: How long to wait between retries :returns: stdout and stderr as string :raises: subprocess.CalledProcessError if command fails more than max_retries - :raises: subprocess.TimeoutExpired if timeout is reached and the command failed more than max_retries + :raises: subprocess.TimeoutExpired if timeout is reached and the command failed more + than max_retries :raises: FileNotFoundError if command is not found """ diff --git a/tests/test_status.py b/tests/test_status.py index d9d8111..072bb57 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -148,11 +148,11 @@ def test_follow_progress_with_qstat_failures(caplog): # Patch subprocess.run in the correct module with patch("hpce_utils.shell.subprocess.run", side_effect=side_effects) as mock_subprocess: # Patch tqdm to avoid actual progress bars in test output - with patch("hpce_utils.managers.uge.status.tqdm"): - with caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"): - status.follow_progress( - username="username", job_ids=["12345678"], update_interval=0.1 - ) + with ( + patch("hpce_utils.managers.uge.status.tqdm"), + caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"), + ): + status.follow_progress(username="username", job_ids=["12345678"], update_interval=0.1) assert mock_subprocess.call_count == 9 @@ -219,11 +219,13 @@ def test_follow_progress_with_initial_qstat_failures(caplog): # Patch subprocess.run in the correct module with patch("hpce_utils.shell.subprocess.run", side_effect=side_effects) as mock_subprocess: # Patch tqdm to avoid actual progress bars in test output - with patch("hpce_utils.managers.uge.status.tqdm"): - with caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"): - status.follow_progress( - username="username", job_ids=["12345678"], update_interval=0.1, exit_after=5 - ) + with ( + patch("hpce_utils.managers.uge.status.tqdm"), + caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"), + ): + status.follow_progress( + username="username", job_ids=["12345678"], update_interval=0.1, exit_after=5 + ) assert mock_subprocess.call_count == 11 diff --git a/tests/uge/test_monitoring.py b/tests/uge/test_monitoring.py index 136965d..ef7fe06 100644 --- a/tests/uge/test_monitoring.py +++ b/tests/uge/test_monitoring.py @@ -11,7 +11,6 @@ def test_cluster_usage(): - usage = get_cluster_usage() # Returns dict mapping username to slot count diff --git a/tests/uge/test_qstat.py b/tests/uge/test_qstat.py index a1338d9..20c3306 100644 --- a/tests/uge/test_qstat.py +++ b/tests/uge/test_qstat.py @@ -18,7 +18,6 @@ def test_all(): - print() jobs_json = get_all_jobs_json() @@ -35,7 +34,6 @@ def test_all(): def test_jobinfo_from_joblist_text(): - jobs = get_all_jobs_text() df = pd.DataFrame(jobs) print(df) From f40a245df281c81e4d68f2df91fc11db1bea5c3d Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 14:44:50 +0100 Subject: [PATCH 3/9] Added jinja2 and tqdm dependencies, pandas dev dependency, improved test skip handling for LMOD and UGE. --- pyproject.toml | 6 +++++- tests/test_lmod.py | 4 +++- tests/uge/test_submitting.py | 5 +++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3832425..8edf91b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,10 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -dependencies = [] +dependencies = [ + "jinja2", + "tqdm", +] [project.optional-dependencies] dev = [ @@ -29,6 +32,7 @@ dev = [ "pre-commit", "pytest", "pytest-cov", + "pandas", "twine", "build", "monkeytype", diff --git a/tests/test_lmod.py b/tests/test_lmod.py index e9567b0..4d2d080 100644 --- a/tests/test_lmod.py +++ b/tests/test_lmod.py @@ -5,7 +5,9 @@ from hpc_funcs import lmod -if not lmod.get_lmod_executable(): +try: + lmod.get_lmod_executable() +except RuntimeError: pytest.skip("Could not find LMOD executable", allow_module_level=True) diff --git a/tests/uge/test_submitting.py b/tests/uge/test_submitting.py index dbc8440..bee7c93 100644 --- a/tests/uge/test_submitting.py +++ b/tests/uge/test_submitting.py @@ -2,8 +2,13 @@ from pathlib import Path import pandas as pd +import pytest +from hpc_funcs.schedulers.uge import has_uge from hpc_funcs.schedulers.uge.constants import TASK_ENVIRONMENT_VARIABLE + +if not has_uge(): + pytest.skip("Could not find UGE executable", allow_module_level=True) from hpc_funcs.schedulers.uge.monitoring import wait_for_jobs from hpc_funcs.schedulers.uge.qacct import get_job_accounting from hpc_funcs.schedulers.uge.qdel import delete_job From 8f172baedd344a1673fec0580def8a5855766c0d Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 14:52:22 +0100 Subject: [PATCH 4/9] Updated shell import to hpc_funcs, removed test_status.py, added UGE skip check. --- tests/test_shell.py | 3 +- tests/test_status.py | 284 -------------------------- tests/uge/test_submitting_progress.py | 6 + 3 files changed, 8 insertions(+), 285 deletions(-) delete mode 100644 tests/test_status.py diff --git a/tests/test_shell.py b/tests/test_shell.py index d6c52bb..18dbf03 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -1,7 +1,8 @@ import subprocess import pytest -from hpce_utils import shell + +from hpc_funcs import shell def test_subprocess_error(): diff --git a/tests/test_status.py b/tests/test_status.py deleted file mode 100644 index 072bb57..0000000 --- a/tests/test_status.py +++ /dev/null @@ -1,284 +0,0 @@ -import os -from pathlib import Path -from subprocess import CalledProcessError as CPError -from unittest.mock import MagicMock, patch - -import pytest -from hpce_utils.managers.uge import status, submitting - -pytest.skip(allow_module_level=True) - -VALID_QSTAT_OUTPUT_RUNNING = """ -job-ID prior name user state submit/start at queue jclass slots ja-task-ID ---------------------------------------------------------------------------------- -12345678 0.5019 job username r 05/19/2025 09:18:15 some.q@some.server.eu. 1 1 -""" -QSTATJ_OUTPUT = """ -job_number: 26936216 -submission_time: 05/19/2025 13:37:07.436 -job-array tasks: 0-1 -""" -VALID_QSTAT_OUTPUT_FINISHED = "" -QSTATJ_OUTPUT_FINISHED = "" -QACCTJ_OUTPUT_FINISHED = """ -============================================================== -qname some.q -hostname some.host -group some_group -owner username -project NONE -department some_department -jobname some_name -jobnumber 12345678 -taskid 1 -pe_taskid NONE -""" - - -def test_qstat_error(): - with patch("hpce_utils.shell.subprocess.run") as mock_subprocess: - mock_subprocess.side_effect = CPError("fail", 1) - with pytest.raises(CPError): - status.get_qstat("username", max_retries=0) - - -def test_follow_progress(): - # Prepare mock process objects - mock_proc_running = MagicMock() - mock_proc_running.stdout = VALID_QSTAT_OUTPUT_RUNNING - mock_proc_running.stderr = "" - mock_proc_running.returncode = 0 - - mock_proc_qstatj = MagicMock() - mock_proc_qstatj.stdout = QSTATJ_OUTPUT - mock_proc_qstatj.stderr = "" - mock_proc_qstatj.returncode = 0 - - mock_proc_finished = MagicMock() - mock_proc_finished.stdout = VALID_QSTAT_OUTPUT_FINISHED - mock_proc_finished.stderr = "" - mock_proc_finished.returncode = 0 - - qstatj_finished_error = CPError( - cmd="qstat -j 12345678", - returncode=1, - stderr="Following jobs do not exist or permissions are not sufficient: 12345678", - output="", - ) - - qacctj_finished = MagicMock() - qacctj_finished.stdout = QACCTJ_OUTPUT_FINISHED - qacctj_finished.stderr = "" - qacctj_finished.returncode = 0 - - side_effects = [ - mock_proc_running, - mock_proc_qstatj, - mock_proc_running, - mock_proc_finished, - qstatj_finished_error, - qacctj_finished, - qstatj_finished_error, # for array_bar.log_errors() - ] - - # Patch subprocess.run - with patch("hpce_utils.shell.subprocess.run", side_effect=side_effects) as mock_subprocess: - # Patch tqdm to avoid actual progress bars in test output - with patch("hpce_utils.managers.uge.status.tqdm"): - status.follow_progress( - username="username", - job_ids=["12345678"], - update_interval=0.1, - ) - - assert mock_subprocess.call_count == 7 - - -def test_follow_progress_with_qstat_failures(caplog): - # Simulate: success, success, failure, success, finished - - # Prepare mock process objects - mock_proc_running = MagicMock() - mock_proc_running.stdout = VALID_QSTAT_OUTPUT_RUNNING - mock_proc_running.stderr = "" - mock_proc_running.returncode = 0 - - mock_proc_qstatj = MagicMock() - mock_proc_qstatj.stdout = QSTATJ_OUTPUT - mock_proc_qstatj.stderr = "" - mock_proc_qstatj.returncode = 0 - - mock_proc_finished = MagicMock() - mock_proc_finished.stdout = VALID_QSTAT_OUTPUT_FINISHED - mock_proc_finished.stderr = "" - mock_proc_finished.returncode = 0 - - qstat_error = CPError( - cmd="qstat", - returncode=1, - stderr="qstat error", - output="", - ) - - qstatj_finished_error = CPError( - cmd="qstat -j 12345678", - returncode=1, - stderr="Following jobs do not exist or permissions are not sufficient: 12345678", - output="", - ) - - qacctj_finished = MagicMock() - qacctj_finished.stdout = QACCTJ_OUTPUT_FINISHED - qacctj_finished.stderr = "" - qacctj_finished.returncode = 0 - - # The sequence: running, qtstat -j, running, error, running, finished - side_effects = [ - mock_proc_running, - mock_proc_qstatj, - mock_proc_running, - qstat_error, - mock_proc_running, - mock_proc_finished, - qstatj_finished_error, - qacctj_finished, - qstatj_finished_error, # for array_bar.log_errors() - ] - - # Patch subprocess.run in the correct module - with patch("hpce_utils.shell.subprocess.run", side_effect=side_effects) as mock_subprocess: - # Patch tqdm to avoid actual progress bars in test output - with ( - patch("hpce_utils.managers.uge.status.tqdm"), - caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"), - ): - status.follow_progress(username="username", job_ids=["12345678"], update_interval=0.1) - - assert mock_subprocess.call_count == 9 - - # Check that the log contains the expected messages - all_logs = caplog.text - assert "stdout" in all_logs - assert "stderr" in all_logs - assert "returncode" in all_logs - - -def test_follow_progress_with_initial_qstat_failures(caplog): - # Simulate: success, success, failure, success, finished - - # Prepare mock process objects - mock_proc_running = MagicMock() - mock_proc_running.stdout = VALID_QSTAT_OUTPUT_RUNNING - mock_proc_running.stderr = "" - mock_proc_running.returncode = 0 - - mock_proc_qstatj = MagicMock() - mock_proc_qstatj.stdout = QSTATJ_OUTPUT - mock_proc_qstatj.stderr = "" - mock_proc_qstatj.returncode = 0 - - mock_proc_finished = MagicMock() - mock_proc_finished.stdout = VALID_QSTAT_OUTPUT_FINISHED - mock_proc_finished.stderr = "" - mock_proc_finished.returncode = 0 - - qstat_error = CPError( - cmd="qstat", - returncode=1, - stderr="qstat error", - output="", - ) - - qstatj_finished_error = CPError( - cmd="qstat -j 12345678", - returncode=1, - stderr="Following jobs do not exist or permissions are not sufficient: 12345678", - output="", - ) - - qacctj_finished = MagicMock() - qacctj_finished.stdout = QACCTJ_OUTPUT_FINISHED - qacctj_finished.stderr = "" - qacctj_finished.returncode = 0 - - # The sequence: error, error, running, qstat -j, running, error, running, finished - side_effects = [ - qstat_error, - qstat_error, - mock_proc_running, - mock_proc_qstatj, - mock_proc_running, - qstat_error, - mock_proc_running, - mock_proc_finished, - qstatj_finished_error, - qacctj_finished, - qstatj_finished_error, # for array_bar.log_errors() - ] - - # Patch subprocess.run in the correct module - with patch("hpce_utils.shell.subprocess.run", side_effect=side_effects) as mock_subprocess: - # Patch tqdm to avoid actual progress bars in test output - with ( - patch("hpce_utils.managers.uge.status.tqdm"), - caplog.at_level("WARNING", logger="hpce_utils.managers.uge.status"), - ): - status.follow_progress( - username="username", job_ids=["12345678"], update_interval=0.1, exit_after=5 - ) - - assert mock_subprocess.call_count == 11 - - # Check that the log contains the expected messages - all_logs = caplog.text - assert "stdout" in all_logs - assert "stderr" in all_logs - assert "returncode" in all_logs - - -def test_wait_for_jobs_using_hold_job(home_tmp_path: Path): - print("scratch:", home_tmp_path) - log_dir = home_tmp_path / "uge_testlogs" - script_1: str = submitting.generate_taskarray_script( - "sleep 10", - cores=1, - cwd=home_tmp_path, - log_dir=log_dir, - name="TestJob", - task_concurrent=1, - task_stop=1, - ) - - script_2: str = submitting.generate_taskarray_script( - "sleep 9", - cores=1, - cwd=home_tmp_path, - log_dir=log_dir, - name="TestJob2", - task_concurrent=1, - task_stop=1, - ) - - job_id_1, _ = submitting.submit_script(script_1, scr=home_tmp_path) - job_id_2, _ = submitting.submit_script(script_2, scr=home_tmp_path) - assert job_id_1 is not None - assert job_id_2 is not None - - print(job_id_1, job_id_2) - - finished_file = status.wait_for_jobs_using_hold_job( - [job_id_1, job_id_2], - scr=home_tmp_path, - log_dir=log_dir, - update_interval=5, - ) - - assert finished_file.exists() - - username = os.getenv("USER") - assert username is not None - qstat, log_str = status.get_qstat(username, max_retries=0) - print(log_str) - - assert job_id_1 not in qstat["job"] - assert job_id_2 not in qstat["job"] diff --git a/tests/uge/test_submitting_progress.py b/tests/uge/test_submitting_progress.py index 1a8f72d..f47f620 100644 --- a/tests/uge/test_submitting_progress.py +++ b/tests/uge/test_submitting_progress.py @@ -1,7 +1,13 @@ import io from pathlib import Path +import pytest + +from hpc_funcs.schedulers.uge import has_uge from hpc_funcs.schedulers.uge.constants import TASK_ENVIRONMENT_VARIABLE + +if not has_uge(): + pytest.skip("Could not find UGE executable", allow_module_level=True) from hpc_funcs.schedulers.uge.monitoring import wait_for_jobs from hpc_funcs.schedulers.uge.monitoring.follow import TaskarrayProgress from hpc_funcs.schedulers.uge.qsub import submit_script, write_script From ee82c4ba71bbe358942d5b107019df3afd11ff01 Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 15:37:03 +0100 Subject: [PATCH 5/9] Removed Development Status classifier from pyproject.toml. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8edf91b..80b200a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ readme = "README.md" license = {text = "MIT"} requires-python = ">=3.11" classifiers = [ - "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", From 45d9aebcdcaa51d64b5efa5f9fe718e9ce57ee3d Mon Sep 17 00:00:00 2001 From: Jimmy Charnley Kromann Date: Fri, 27 Mar 2026 15:44:01 +0100 Subject: [PATCH 6/9] Removed empty test_slurm.py file. --- tests/test_slurm.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/test_slurm.py diff --git a/tests/test_slurm.py b/tests/test_slurm.py deleted file mode 100644 index e69de29..0000000 From 5bafde937e7f7c5785ff89378f7cbb412f902401 Mon Sep 17 00:00:00 2001 From: Jimmy Kromann Date: Sat, 28 Mar 2026 12:41:36 +0100 Subject: [PATCH 7/9] Removed code-quality workflow, simplified publish workflow with make targets. --- .github/workflows/code-quality.yml | 41 ------------------------------ .github/workflows/publish.yml | 6 ++--- .github/workflows/test.yml | 7 +++-- Makefile | 6 ----- 4 files changed, 8 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/code-quality.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index fa8a1a4..0000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Code Quality - -on: - push: - branches: [main] - pull_request: - branches: [main] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - code-quality: - name: Lint & Type Check - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ruff mypy - - - name: Check formatting with Ruff - run: ruff format --check src/ tests/ - - - name: Lint with Ruff - run: ruff check src/ tests/ - - - name: Type check with mypy - run: mypy src/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb396b0..e89169d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,13 +23,13 @@ jobs: pip install build twine - name: Build package - run: python -m build + run: make build - name: Check distribution - run: twine check dist/* + run: make test-dist - name: Publish to PyPI env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: twine upload dist/* + run: make upload diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 390ea6a..1060af5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,11 +31,14 @@ jobs: - name: Run tests run: source ./env/bin/activate && make test python=python + - name: Check format and types + run: make format + - name: Build package - run: make build + run: source ./env/bin/activate && make build - name: Check distribution - run: make test-dist + run: source ./env/bin/activate && make test-dist - name: Test installation from wheel run: | diff --git a/Makefile b/Makefile index 0da135a..1f31fe7 100644 --- a/Makefile +++ b/Makefile @@ -25,12 +25,6 @@ env_uv: uv pip install -e .[dev] --python ${env}/bin/python ${python} -m pre_commit install -env_conda: - conda env create -f ./environment.yml -p ${env} --quiet - ${python} -m pre_commit install - ${python} -m pip install -e . - - ## Development update-format: From 7e592716808a49a226743c42d3c7c355aeb3f688 Mon Sep 17 00:00:00 2001 From: Jimmy Kromann Date: Sat, 28 Mar 2026 12:55:35 +0100 Subject: [PATCH 8/9] Updated ty pre-commit hook to use system language. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66ceea6..f7a4c35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: name: Static type checking entry: ty check files: ^src/.*\.py$ - language: python + language: system - id: jupyisort name: Sorts ipynb imports From ff88cbae4546350509a983ed020630d1046ac35e Mon Sep 17 00:00:00 2001 From: Jimmy Kromann Date: Sat, 28 Mar 2026 12:57:38 +0100 Subject: [PATCH 9/9] Updated format check to use activated virtual environment. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1060af5..2e0fad9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: run: source ./env/bin/activate && make test python=python - name: Check format and types - run: make format + run: source ./env/bin/activate && make format - name: Build package run: source ./env/bin/activate && make build