Release v1.2.0

.github/workflows/test.yml

@@ -19,4 +19,4 @@ jobs:
     with:
       local_package_name: hyp3_mintpy
       python_versions: >-
-        ["3.10", "3.11", "3.12", "3.13"]
+        ["3.10", "3.11", "3.12", "3.13", "3.14"]

CHANGELOG.md

@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.0]
+
+### Added
+- Added new parameters `--publish-bucket` and `--publish-prefix` to upload products to s3 bucket.
+- Added parameter `--input-bucket` to pull multiburst products from an specific s3 bucket.
+- Added network and coherence plots to the output product.
+- Added check for valid pixels on the downloaded pairs, if it does not find any it removes the pair.
+
+### Changed
+- Renamed `--prefix` parameter to `--input-prefix` to pull product from an specific s3 prefix.
+- Restricted numpy to <2.4
+- Changed `rename_products` function to find nan baselines and change them for 0.
+
 ## [1.1.0]
 
 ### Added

environment.yml

@@ -15,6 +15,7 @@ dependencies:
   - hyp3lib>=3,<4
   - hyp3_sdk
   - mintpy
+  - numpy<2.4
   - rasterio
   # TODO: insert conda-forge dependencies as list here
   - pip:

pyproject.toml

@@ -21,6 +21,7 @@ classifiers=[
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
     "hyp3lib>=3,<5",

requirements-static.txt

@@ -1,3 +1,3 @@
-ruff==0.14.5
-mypy==1.18.2
+ruff==0.15.12
+mypy==2.1.0
 opensarlab_lib

src/hyp3_mintpy/__main__.py

@@ -10,6 +10,7 @@
 from hyp3lib.fetch import write_credentials_to_netrc_file
 
 from hyp3_mintpy.process import process_mintpy
+from hyp3_mintpy.util import nullable_string, upload_file_to_s3_with_publish_access_keys
 
 
 def main() -> None:
@@ -19,15 +20,37 @@ def main() -> None:
     parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
 
     # TODO: Your arguments here
-    parser.add_argument('--job-name', help='The name of the HyP3 job', required=False)
+    parser.add_argument('--project-name', type=nullable_string, help='The name of the HyP3 project', required=False)
+    parser.add_argument('--start-date', type=nullable_string, help='Start date for the timeseries (YYYY-MM-DD)')
+    parser.add_argument('--end-date', type=nullable_string, help='End date for the timeseries (YYYY-MM-DD)')
     parser.add_argument(
-        '--prefix', help='Folder that contains multiburst products in the volcsarvatory bucket', required=False
+        '--min-coherence', default=0.01, type=float, help='The minimum coherence to process', required=False
     )
     parser.add_argument(
-        '--min-coherence', default=0.01, type=float, help='The minimum coherence to process', required=False
+        '--input-bucket',
+        type=nullable_string,
+        default=None,
+        help='Bucket to pull multiburst products',
+    )
+    parser.add_argument(
+        '--input-prefix',
+        type=nullable_string,
+        default=None,
+        help='Prefix to pull the products from the publish bucket',
+    )
+    parser.add_argument(
+        '--publish-bucket',
+        type=nullable_string,
+        default=None,
+        help='Additionally, publish products to this bucket. Necessary credentials must be provided '
+        'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.',
+    )
+    parser.add_argument(
+        '--publish-prefix',
+        type=nullable_string,
+        default=None,
+        help='Prefix for the bucket where the products will be published',
     )
-    parser.add_argument('--start-date', type=str, help='Start date for the timeseries (YYYY-MM-DD)')
-    parser.add_argument('--end-date', type=str, help='End date for the timeseries (YYYY-MM-DD)')
 
     args = parser.parse_args()
 
@@ -47,8 +70,9 @@ def main() -> None:
         )
 
     product_file = process_mintpy(
-        job_name=args.job_name,
-        prefix=args.prefix,
+        project_name=args.project_name,
+        input_bucket=args.input_bucket,
+        input_prefix=args.input_prefix,
         min_coherence=args.min_coherence,
         start=args.start_date,
         end=args.end_date,
@@ -57,6 +81,19 @@ def main() -> None:
     if args.bucket:
         upload_file_to_s3(product_file, args.bucket, args.bucket_prefix)
 
+    if args.publish_bucket:
+        if args.publish_prefix:
+            prefix = args.publish_prefix
+            s3_name = str(product_file)
+        else:
+            if args.start_date is None or args.end_date is None:
+                prefix = 'multiburst_ts/'
+                s3_name = str(product_file)
+            else:
+                prefix = f'multiburst_ts/{str(product_file).split(".zip")[0]}'
+                s3_name = f'{args.start_date.replace("-", "")}_{args.end_date.replace("-", "")}.zip'
+        upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix, s3_name)
+
 
 if __name__ == '__main__':
     main()

src/hyp3_mintpy/process.py

@@ -44,6 +44,14 @@ def rename_products(folder: str) -> None:
         ar = txts[0].open()
         lines = ar.readlines()
         ar.close()
+        nan = any(['Baseline' in line and 'nan' in line for line in lines])
+        if nan:
+            ar = txts[0].open('w')
+            for line in lines:
+                if 'Baseline' in line and 'nan' in line:
+                    line = line.replace('nan', '0')
+                ar.write(line)
+            ar.close()
         burst = lines[0].split('_')[1] + '_' + lines[0].split('_')[2]
         for f in fs:
             name = f.name
@@ -83,6 +91,9 @@ def download_job_pairs(
     for z in file_list:
         if check_product(z.name, start, end):
             shutil.unpack_archive(str(z), folder)
+            wphase = list(Path(folder).glob('**/*_wrapped_phase.tif'))[0]
+            if not util.check_valid_pixels(wphase):
+                shutil.rmtree('/'.join(str(wphase).split('/')[0:-1]))
         z.unlink()
 
     rename_products(folder)
@@ -91,31 +102,33 @@ def download_job_pairs(
 
 
 def download_bucket_pairs(
-    key: str | None = None,
+    bucket: str,
+    key: str = '',
     start: str | None = None,
     end: str | None = None,
-    path: str = 'multiburst_products/',
-    bucket: str = 'volcsarvatory-data-test',
 ) -> str:
     """Downloads multiburst products from bucket and renames files to meet MintPy standards.
 
     Args:
+        bucket: Name of the bucket.
         key: Folder name that contains the multiburst product.
         start: Start date for the timeseries if one of the product dates is before this, it won't be downloaded.
         end: End date for the timeseries if one of the product dates is after this, it won't be downloaded.
         path: Additional prefix to the products.
-        bucket: Name of the bucket.
     """
     s3 = boto3.resource('s3', config=boto3.session.Config(signature_version=botocore.UNSIGNED))
     buck = s3.Bucket(bucket)
-    folder = str(key).split('/')[-1]
+    folder = f'{str(key).split("/")[-1]}'
     Path.mkdir(Path(folder))
-    for s3_object in tqdm(buck.objects.filter(Prefix=f'{path}{key}')):
+    for s3_object in tqdm(buck.objects.filter(Prefix=f'{key}')):
         path, filename = os.path.split(s3_object.key)
         if check_product(filename, start, end):
             buck.download_file(s3_object.key, f'{folder}/{filename}')
             z = Path(f'{folder}/{filename}')
             shutil.unpack_archive(str(z), folder)
+            wphase = list(Path(folder).glob('**/*_wrapped_phase.tif'))[0]
+            if not util.check_valid_pixels(wphase):
+                shutil.rmtree('/'.join(str(wphase).split('/')[0:-1]))
             z.unlink()
     rename_products(folder)
 
@@ -300,36 +313,47 @@ def run_mintpy(output_name: str) -> Path:
     subprocess.call(f'mv {output_name}/MintPy/*.h5 {output_name}/', shell=True)
     subprocess.call(f'mv {output_name}/MintPy/inputs/geometry*.h5 {output_name}/', shell=True)
     subprocess.call(f'mv {output_name}/MintPy/*.txt {output_name}/', shell=True)
+    subprocess.call(f'mv {output_name}/MintPy/*.pdf {output_name}/', shell=True)
+    subprocess.call(f'mv {output_name}/MintPy/*.png {output_name}/', shell=True)
     subprocess.call(f'rm -rf {output_name}/MintPy {output_name}/S1_* {output_name}/shape_*', shell=True)
     output_zip = shutil.make_archive(base_name=output_name, format='zip', base_dir=output_name)
 
     return Path(output_zip)
 
 
 def process_mintpy(
-    job_name: str | None, prefix: str | None, min_coherence: float, start: str | None = None, end: str | None = None
+    project_name: str | None,
+    input_bucket: str | None,
+    input_prefix: str | None,
+    min_coherence: float,
+    start: str | None = None,
+    end: str | None = None,
 ) -> Path:
     """Create a greeting product.
 
     Args:
-        job_name: Name of the HyP3 project.
-        prefix: Folder that contains multiburst products.
+        project_name: Name of the HyP3 project.
+        input_bucket: Bucket that contains multiburst products.
+        input_prefix: Folder that contains multiburst products.
         min_coherence: Minimum coherence for timeseries processing.
         start: Start date for the timeseries
         end: End date for the timeseries
 
     Returns:
         Path for the output zip file.
     """
-    if job_name is None and prefix is None:
-        raise ValueError('You should give a job name or a bucket to pull the data from')
-    elif job_name is not None and prefix is not None:
-        warnings.warn('Both job name and prefix were given. You should give just one. Using job name...')
-
-    if job_name is not None:
-        output_name = download_job_pairs(job_name, start, end)
+    if project_name is None and (input_bucket is None or input_prefix is None):
+        raise ValueError('You should give a project name or a bucekt and a prefix to pull the data')
+    elif project_name is not None and input_prefix is not None:
+        warnings.warn('Both job name and prefix were given. You should give just one. Using input prefix...')
+
+    if input_bucket is not None:
+        if input_prefix is None:
+            input_prefix = ''
+        output_name = download_bucket_pairs(str(input_bucket), input_prefix, start, end)
     else:
-        output_name = download_bucket_pairs(prefix, start, end)
+        output_name = download_job_pairs(str(project_name), start, end)
+
     set_same_frame(output_name, wgs84=True)
 
     write_cfg(output_name, str(min_coherence))

src/hyp3_mintpy/util.py

@@ -6,10 +6,12 @@
 from datetime import datetime
 from pathlib import Path
 
+import boto3
 import geopandas as gpd
 import numpy as np
 import rasterio
 import shapely.wkt
+from hyp3lib.aws import get_content_type, get_tag_set
 from mintpy.utils import readfile
 from osgeo import gdal, ogr, osr
 from pyproj import Transformer
@@ -280,3 +282,69 @@ def save_shapefile(
     feat = None
 
     ds = layer = feat = None
+
+
+def nullable_string(argument_string: str) -> str | None:
+    """Identify if string is None.
+
+    Takes:
+    argument_string: Input string.
+
+    Returns: None if input string is 'None' else input string
+    """
+    argument_string = argument_string.replace('None', '').strip()
+    return argument_string if argument_string else None
+
+
+def check_valid_pixels(image_path: Path) -> bool:
+    """Check if the image has valid pixels.
+
+    Takes:
+        image: Local image path
+
+    Returns: True if there's at least one valid pixel, False otherwise
+    """
+    has_valid = False
+    with rasterio.open(str(image_path)) as src:
+        data = src.read(1)
+        nodata = src.nodata
+        valid_mask = ~np.isnan(data) & ~np.isinf(data)
+        if nodata is not None:
+            valid = data != nodata
+            valid_mask &= valid
+        has_valid = bool(np.any(valid_mask))
+    return has_valid
+
+
+def upload_file_to_s3_with_publish_access_keys(
+    path_to_file: Path, bucket: str, prefix: str = '', s3_name: str | None = None
+) -> None:
+    """Uploads file to s3 bucket.
+
+    path_file: Local file path
+    bucket: Bucket name
+    prefix: Bucket prefix
+    s3_name: Name of file in s3 bucekt
+    """
+    try:
+        access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID']
+        access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY']
+    except KeyError:
+        raise ValueError(
+            'Please provide S3 Bucket upload access key credentials via the '
+            'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables'
+        )
+
+    s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret)
+
+    if s3_name is None:
+        s3_name = path_to_file.name
+    key = str(Path(prefix) / s3_name)
+
+    extra_args = {'ContentType': get_content_type(key)}
+
+    s3_client.upload_file(str(path_to_file), bucket, key, extra_args)
+
+    tag_set = get_tag_set(path_to_file.name)
+
+    s3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set)