From 62831c02bd4214a3e3fc0ccb5d4c176e67a8bb5c Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 16 Mar 2026 15:03:50 -0800 Subject: [PATCH 01/21] Clipping multimodal data --- .gitignore | 11 + README.md | 87 ++--- src/satchip/generate_chips.py | 439 +++++++++++++++++++++++ src/satchip/hls.py | 170 +++++++++ src/satchip/modality.py | 10 + src/satchip/mosaic.py | 227 ++++++++++++ src/satchip/{ => old}/chip_data.py | 60 +++- src/satchip/{ => old}/chip_hls.py | 42 ++- src/satchip/{ => old}/chip_hyp3s1rtc.py | 0 src/satchip/{ => old}/chip_label.py | 0 src/satchip/{ => old}/chip_operas1rtc.py | 47 ++- src/satchip/{ => old}/chip_sentinel2.py | 60 +++- src/satchip/{ => old}/chip_view.py | 0 src/satchip/{ => old}/chip_xr_base.py | 0 src/satchip/{ => old}/major_tom_grid.py | 0 src/satchip/{ => old}/terra_mind_grid.py | 0 src/satchip/old/utils.py | 116 ++++++ src/satchip/opera_rtc.py | 106 ++++++ tests/test_chip_hyp3s1rtc.py | 172 --------- tests/test_chip_sentinel2.py | 24 -- tests/test_integration.py | 59 --- tests/test_stub.py | 2 - tests/test_terra_mind_grid.py | 23 -- 23 files changed, 1258 insertions(+), 397 deletions(-) create mode 100644 src/satchip/generate_chips.py create mode 100644 src/satchip/hls.py create mode 100644 src/satchip/modality.py create mode 100644 src/satchip/mosaic.py rename src/satchip/{ => old}/chip_data.py (75%) rename src/satchip/{ => old}/chip_hls.py (84%) rename src/satchip/{ => old}/chip_hyp3s1rtc.py (100%) rename src/satchip/{ => old}/chip_label.py (100%) rename src/satchip/{ => old}/chip_operas1rtc.py (80%) rename src/satchip/{ => old}/chip_sentinel2.py (85%) rename src/satchip/{ => old}/chip_view.py (100%) rename src/satchip/{ => old}/chip_xr_base.py (100%) rename src/satchip/{ => old}/major_tom_grid.py (100%) rename src/satchip/{ => old}/terra_mind_grid.py (100%) create mode 100644 src/satchip/old/utils.py create mode 100644 src/satchip/opera_rtc.py delete mode 100644 tests/test_chip_hyp3s1rtc.py delete mode 100644 tests/test_chip_sentinel2.py delete mode 100644 tests/test_integration.py delete mode 100644 tests/test_stub.py delete mode 100644 tests/test_terra_mind_grid.py diff --git a/.gitignore b/.gitignore index da270fc..4dc2ca6 100644 --- a/.gitignore +++ b/.gitignore @@ -246,3 +246,14 @@ tags *tif* integration_test/ chips +*.h5 + +*.cpg +*.dbf +*.prj +*.sbn +*.sbx +*.shp +*.shx +*.zip +*.qgz diff --git a/README.md b/README.md index 19d7c7d..3a05206 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,30 @@ # SatChip -A package for satellite image AI data prep. This package "chips" data labels and satellite imagery into 264x264 image arrays following the TerraMind extension of the MajorTom specification. +A package for satellite image AI data prep. ## Usage -`SatChip` relies on a two-step process; chip your label train data inputs, then create corresponding chips for different remote sensing data sources. -### Step 1: Chip labels -The `chiplabel` CLI tool takes a GDAL-compatible image, a collection date, and an optional chip directory as input using the following format: +```python +import satchip -```bash -chiplabel PATH/TO/LABELS.tif DATE(UTC FORMAT) --chipdir CHIP_DIR -``` -For example: -```bash -chiplabel LA_damage_20250113_v0.tif 2024-01-01T01:01:01 --chipdir chips -``` -This will produce an output zipped Zarr store label dataset with the name `{LABEL}_{SAMPLE}.zarr.zip` (see the (Tiling Schema)[#tiling_schema] section for details on the `SAMPLE` name) to the `LABEL` directory in the specified chip directory (`--chipdir`). This file will be the input to the remote sensing data chipping step. - -For more information on usage see `chiplabel --help` - -### Step 2: Chip remote sensing data -The `chipdata` CLI tool takes a path to a directory containing chip labels, a dataset name, a date range and a set of optional parameters using the following format: -```bash -chipdata PATH/TO/LABEL DATASET Ymd-Ymd \ - --maxcloudpct MAX_CLOUD_PCT --strategy STRATEGY \ - --chipdir CHIPPUT_DIR --imagedir IMAGE_DIR -``` -For example: -```bash -chipdata LABEL S2L2A 20250112-20250212 --maxcloudpct 20 --chipdir CHIP_DIR --imagedir IMAGES -``` -Similarly to step 1, this will produce an output zipped Zarr store that contains chipped data for your chosen dataset with the name `{LABELS_{SAMPLE}_{DATASET}.zarr.zip`. The arguments are as follows: -- `PATH/TO/LABEL`: the path to your training labels -- `DATASET`: The satellite imagery dataset you would like to create labels for. See the list below for all current options. -- `Ymd-Ymd`: The date range to select imagery from. For example, `20250112-20250212` selects imagery between January 12 and February 12, 2025. -- `MAX_CLOUD_PCT`: For optical data, this optional parameter lets you set the maximum amount of cloud coverage allowed in a chip. Values between 0 and 100 are allowed. Cloud coverage is calculated on a per-chip basis. The default is 100 i.e., no limit. -- `STRATEGY`: Lets you selected what data inside your date range will be used to create chips. Specifying `BEST` (the default) will create a chip for the image closest to the beginning of your date range that has at least 95% spatial coverage. Specifying `ALL` will create chips for all images within your date range that have at least 95% spatial coverage. -- `CHIP_DIR`: Specifies the directory where the image chips will be saved. If not specified, this defaults to your current directory. -- `IMAGE_DIR`: Specifies the directory where the full-size satellite images will be downloaded to. If this argument is not provided, the images will be stored in the `IMAGES` directory within `CHIP_DIR`. - -Currently supported datasets include: -- `S2L2A`: Sentinel-2 L2A data sourced from the [Sentinel-2 AWS Open Data Archive](https://registry.opendata.aws/sentinel-2/) -- `HLS`: Harmonized Landsat Sentinel-2 data sourced from [LP DAAC's Data Archive](https://www.earthdata.nasa.gov/data/projects/hls) -- `S1RTC`: OPERA Sentinel-1 Radiometric Terrain Corrected (RTC) data from [ASF DAAC's Data Archive](https://www.jpl.nasa.gov/go/opera/products/rtc-product/) -- `HYP3S1RTC`: Sentinel-1 Radiometric Terrain Corrected (RTC) data created using [ASF's HyP3 on-demand platform](https://hyp3-docs.asf.alaska.edu/guides/rtc_product_guide/) - -## Tiling Schema +data_paths = { + MODALITY: hwds_path / MODALITY, + "RAW": modality_path / "RAW", + "WGS84": modality_path / "WGS84", + "MERGE": modality_path / "MERGE", + "CHIPS": modality_path / "CHIPS", + "CHIPS_TM": modality_path / "CHIPS_TM", + "PLOTS": modality_path / "PLOTS", + "SPLITS": modality_path / "SPLITS", +} -This package chips images based on the [TerraMesh grid system](https://huggingface.co/datasets/ibm-esa-geospatial/TerraMesh), which builds on the [MajorTOM grid system](https://github.com/ESA-PhiLab/Major-TOM). +modalities = ['HLS'] -The MajorTOM grid system provides a global set of fixed image grids that are 1068x1068 pixels in size. A MajorTOM grid can be defined for any tile size, but we fix the grid to 10x10 Km tiles. Tiles are named using the format: -``` -ROW[U|D]_COL[L|R] -``` -Where, `ROW` is indexed from the equator, with a suffix `U` (up) for tiles north of the equator and `D` (down) for tiles south of it, and `COL` is indexed from the prime meridian, with a suffix `L` (left) for tiles east of the prime meridian and `R` (right) for tiles west of it. - -To support finer subdivisions, the TerraMesh grid system divides each MajorTOM grid into a 4x4 set of sub-tiles, each 264x264 pixels. The subgrid is centered within the parent tile, leaving a 6-pixel border around each sub-tile. Subgrid names extend the base format with two additional indices: -``` -ROW[U|D]_COL[L|R]_SUBCOL_SUBROW -``` -For instance, the bottom-left subgrid of MajorTOM tile `434U_876L` is named `434U_876L_0_3`. See the figure below for a visual description: - -![TerraMesh tiling schema](assets/satchip_schema.svg) +data = satchip.find_data(area, modality) +reprojected_data = satchip.repoject(raw_data, projection='WGS84', modality=) # stack bands and mosaic +mosaics = satchip.mosaic(data, stack_bands=True) # stack bands and mosaic +masks = satchip.generate_masks(mosaics) -## Viewing Chips -Assessing chips after their creation can be challenging due to the large number of small images created. To address this issue, SatChip includes a `chipview` CLI tool that uses Matplotlib to quickly visualize the data included within the created zipped Zarr stores: -```bash -chipview PATH/TO/CHIP.zarr.zip --band BAND +chips = satchip.chip_data(mosaics, masks) +chips = satchip.filter_chips(chips) ``` -Where `PATH/TO/CHIPS.zarr.zip` is the path to the chip file (labels or image data), and `BAND` is an OPTIONAL name of the band you would like to view. If no band is specified, an OPERA-style RGB decomposition will be shown for RTC data, and an RGB composite will be shown for optical data. - -## License -`SatChip` is licensed under the BSD-3-Clause open source license. See the LICENSE file for more details. - -## Contributing -Contributions to the `SatChip` are welcome! If you would like to contribute, please submit a pull request on the GitHub repository. diff --git a/src/satchip/generate_chips.py b/src/satchip/generate_chips.py new file mode 100644 index 0000000..40ada4e --- /dev/null +++ b/src/satchip/generate_chips.py @@ -0,0 +1,439 @@ +from pathlib import Path +import shutil +import zipfile + + +import cartopy.crs as ccrs +import earthaccess +import gdown +import geopandas as gpd +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import rasterio +from rasterio import features +from rasterio.windows import Window +from shapely.geometry import box +from sklearn.model_selection import train_test_split + +from modality import Modality +import mosaic +import hls +import opera_rtc + +QUITE = True +CHIP_SIZE = 256 +RNG_SEED = 42 + + +MODALITY = "RTC" +ALL_BANDS = ("VV", "VH", "mask") +STACK_BANDS = ("VV", "VH") +CHIP_BANDS = ("BANDS", "EVENT", "MASK") + +MODALITY = "HLS" +ALL_BANDS = ("B", "G", "R", "N", "SW1", "SW2", "Fmask") +STACK_BANDS = ("B", "G", "R", "N", "SW1", "SW2") +CHIP_BANDS = ("BANDS", "EVENT", "MASK", "Fmask") + + +def main(modalities: list[Modality]): + hwds_path = Path("hwds") + + print('Making folders') + data_paths = { + "CHIPS_ALL": hwds_path / "CHIPS_ALL", + "CHIPS": hwds_path / "CHIPS", + "MERGED": hwds_path / "MERGED", + } + + for p in ("CHIPS", "CHIPS_ALL"): + shutil.rmtree(data_paths[p], ignore_errors=True) + + for p in data_paths.values(): + p.mkdir(parents=True, exist_ok=True) + + gdf = _load_event_database_with_buffer(hwds_path) + gdf_utm = gdf.to_crs(32615) + gdf["buffered_event"] = gdf_utm.buffer(3000).to_crs(4326) + gdf["buffered_event_background"] = gdf_utm.buffer(10000).to_crs(4326) + + # keepers = [1442, 622, 1079, 628] + # keepers = [1442, 622] + # gdf = gdf[gdf["swathID"].isin(keepers)] + + earthaccess.login() + + tm_chips = [] + + for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): + swathID = f"{int(swath['swathID']):04d}" + print(f'Processing Swath {swathID} ({i} / {len(gdf)})') + + merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) + + template_path = merged[modalities[0].id]['BANDS'] + event_tif, mask_tif = _generate_masks( + template_path, swathID, swath + ) + + merged_data = { + **merged, + "EVENT": event_tif, + "MASK": mask_tif, + } + + breakpoint() + + if not all(is_valid_data(merged_data, modality) for modality in modalities): + print("Skipping: not enough valid data") + continue + + breakpoint() + + print("Chipping!") + chips = _chip_data(merged_data, data_paths) + + good_chips = filter_chips(chips) + print(f"Found {len(good_chips)} good chips") + + for chip in good_chips: + for band, chip_path in chip.items(): + if band not in ("MASK", "BANDS"): + continue + + dest = data_paths["CHIPS_TM"] / chip_path.name + shutil.copy(chip_path, dest) + + tm_chips += good_chips + + for _, swath in gdf.iterrows(): + swath_id = _make_swath_id(swath["swathID"]) + + merged_file = list(data_paths["MERGED"].glob(f"{swath_id}.*BANDS.tif")) + if len(merged_file) == 0: + print(f"no chips for {swath_id}") + continue + + all_chips = list(data_paths["CHIPS"].glob(f"*.{swath_id}.*.tif")) + good_chips = list(data_paths["CHIPS_TM"].glob(f"*.{swath_id}.*.tif")) + + print(f"plotting {swath_id}") + _plot_chips( + merged_file[0], all_chips, good_chips, swath, save_to=data_paths["PLOTS"] + ) + + band_chips = list(data_paths["CHIPS_TM"].glob("*BANDS.tif")) + + means, stds = calculate_stats(chips=band_chips) + + means_str = ", ".join(f"{x:.4f}" for x in means) + stds_str = ", ".join(f"{x:.4f}" for x in stds) + + stats_str = ( + f"Means {STACK_BANDS}: {means_str}\n" + f"Stds {STACK_BANDS}: {stds_str}\n" + ) + (hwds_path / 'statistics.txt').write_text(stats_str) + print(stats_str) + + +def _generate_masks( + template_data_path: Path, swathID: str, swath: pd.Series +) -> tuple[Path, Path]: + event_path = template_data_path.parent / f"{swathID}.EVENT.tif" + mask_path = template_data_path.parent / f"{swathID}.MASK.tif" + + with rasterio.open(template_data_path) as ds: + profile = ds.profile + + mask_raster = features.rasterize( + shapes=[[swath["geometry"], 1]], + fill=0, + out_shape=ds.shape, + transform=ds.transform, + ) + + with rasterio.open(mask_path, "w", **profile) as dst: + dst.write(mask_raster, 1) + print("generated:", mask_path) + + event_mask = features.rasterize( + shapes=[ + [swath["buffered_event_background"], 3], + [swath["buffered_event"], 2], + [swath["geometry"], 1], + ], + fill=0, + out_shape=ds.shape, + transform=ds.transform, + ) + + with rasterio.open(event_path, "w", **profile) as dst: + dst.write(event_mask, 1) + print("generated:", event_path) + + return event_path, mask_path + + +def _load_event_database_with_buffer(data_dir: Path): + # use 60-swath version + hwds_google_drive_id = "1h_JIEcrrUF3OSTrmwAKNPa0eUEhPA2Xx" + drive_url = f"https://drive.google.com/uc?id={hwds_google_drive_id}" + + shp_dir = data_dir / "SHP" + shp_dir.mkdir(parents=True, exist_ok=True) + + filename = "hwds_v3_20250205_subset_60.zip" + + zip_path = shp_dir / filename + + if not zip_path.exists(): + gdown.download(drive_url, str(zip_path), quiet=False) + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(path=shp_dir) + + shp_path = shp_dir / "hwds_v3_20250205_subset_60.shp" + gdf = gpd.read_file(shp_path) + + gdf["swathDate"] = pd.to_datetime(gdf["swathDate"], format="%Y-%m-%d") + gdf["ls5hlsDate"] = pd.to_datetime(gdf["ls5hlsDate"], format="%Y-%m-%d") + gdf["s1Date"] = pd.to_datetime(gdf["s1Date"], format="%Y-%m-%d") + + return gdf + + +def create_split_files(band_chips: list[Path], splits_path: Path) -> None: + chip_ids = [p.name.removesuffix("BANDS.tif") for p in band_chips] + + the_rest, test = train_test_split(chip_ids, test_size=0.15, random_state=RNG_SEED) + train, val = train_test_split(the_rest, test_size=0.15, random_state=RNG_SEED) + + splits = {"train": train, "val": val, "test": test} + + for split, chip_ids in splits.items(): + split_path = splits_path / f"{split}.txt" + split_path.write_text("\n".join(chip_ids)) + + +def calculate_stats(chips: list[Path], n_bands: int = 2) -> tuple: + mean = np.zeros(n_bands, dtype=np.float64) + M2 = np.zeros(n_bands, dtype=np.float64) + count = np.zeros(n_bands, dtype=np.float64) + + for chip in chips: + with rasterio.open(chip) as src: + band_data = src.read() + count, mean, M2 = 0, 0, 0 + + _, H, W = band_data.shape + + batch_count = H * W + batch_mean = band_data.mean(axis=(1, 2)) + batch_var = band_data.var(axis=(1, 2)) + + delta = batch_mean - mean + total_count = count + batch_count + + mean = mean + delta * (batch_count / total_count) + M2 = ( + M2 + + batch_var * batch_count + + (delta**2) * count * batch_count / total_count + ) + count = total_count + + variance = M2 / count + std = np.sqrt(variance) + + return mean, std + + +def _plot_chips( + merged_band_file, + all_chips, + good_chips, + swath, + save_to: Path | None = None, + quite=QUITE, +): + crs_pc = ccrs.PlateCarree() + + with rasterio.open(merged_band_file) as ds: + bounds = ds.bounds + full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] + band_data = ds.read() + + img = get_img(band_data) + + # plot BANDS and geom + fig, ax = plt.subplots( + 1, + 1, + subplot_kw={"projection": crs_pc}, + figsize=(12, 12), + layout="constrained", + ) + + swath_geom = swath["geometry"] + + ax.imshow(img, extent=full_extent, origin="upper", transform=crs_pc) + ax.add_geometries( + [swath_geom], edgecolor="red", linewidth=2, facecolor="none", crs=crs_pc + ) + + def show_chips(chips, color, linewidth, z): + for chip in chips: + with rasterio.open(chip) as ds: + chip_bounds = ds.bounds + chip_geom = box( + chip_bounds.left, + chip_bounds.bottom, + chip_bounds.right, + chip_bounds.top, + ) + + ax.add_geometries( + [chip_geom], + edgecolor=color, + linewidth=linewidth, + alpha=1, + zorder=z, + facecolor="none", + crs=crs_pc, + ) + + show_chips(all_chips, "yellow", 1, z=1) + show_chips(good_chips, "blue", 3, z=2) + + ax.set_extent(full_extent, crs=crs_pc) + + if save_to: + plt.savefig( + save_to / f"{merged_band_file.name.removesuffix('BANDS.tif')}.png", + dpi=300, + bbox_inches="tight", + ) + + if not quite: + plt.show() + + plt.close(fig) + + +def _make_swath_id(swathID): + return f"{int(swathID):04d}" + + +def _chip_data(merged: dict[str, Path], data_paths: dict[str, Path], chip_size=CHIP_SIZE): + chips = {} + + grid = [] + with rasterio.open(merged["BANDS"]) as ref: + n_cols = ref.width // chip_size + n_rows = ref.height // chip_size + + for row in range(n_rows): + for col in range(n_cols): + window = Window(col * chip_size, row * chip_size, chip_size, chip_size) + bounds = ref.window_bounds(window) + + tile_id = f"{row:03d}.{col:03d}" + chips[tile_id] = {} + grid.append((tile_id, bounds)) + + for chip_layer in CHIP_BANDS: + layer_path = merged[chip_layer] + + with rasterio.open(layer_path) as src: + for tile_id, bounds in grid: + window = src.window(*bounds) + window = Window( + round(window.col_off), + round(window.row_off), + round(window.width), + round(window.height), + ) + + data = src.read(window=window) + + if chip_layer == 'BANDS': + data = data_transform(data) + + chip_meta = src.meta.copy() + chip_meta.update( + { + "width": window.width, + "height": window.height, + "transform": src.window_transform(window), + } + ) + + chip_name = f"{tile_id}.{layer_path.name}" + chip_path = data_paths["CHIPS"] / chip_name + + with rasterio.open(chip_path, "w", **chip_meta) as dst: + dst.write(data) + + chips[tile_id][chip_layer] = chip_path + + return chips + + +def is_valid_data(merged, modality): + merged_data = merged[modality.id] + + if modality.id == 'HLS': + is_valid = hls.is_valid_hls(merged_data['Fmask'], merged['EVENT']) + print('HLS', is_valid) + elif modality.id == 'RTC': + is_valid = opera_rtc.is_valid_rtc(merged_data['mask'], merged['EVENT']) + print('RTC', is_valid) + + return is_valid + + +def filter_chips(chips): + if MODALITY == 'HLS': + filtered_chips = hls.filter_hls_chips(chips) + elif MODALITY == 'RTC': + filtered_chips = opera_rtc.filter_rtc_chips(chips) + + return filtered_chips + + +def get_img(band_data): + if MODALITY == 'HLS': + img = hls.get_hls_img(band_data) + elif MODALITY == 'RTC': + img = opera_rtc.get_rtc_img(band_data) + + return img + + +def data_transform(data): + if MODALITY == 'RTC': + data = 10 * np.log10(np.clip(data, 1e-10, None)) + + return data + + +if __name__ == "__main__": + opera_rtc_mod = Modality( + id="RTC", + all_bands=("VV", "VH", "mask"), + stack_bands=("VV", "VH"), + chip_bands=("BANDS", "EVENT", "MASK") + ) + + hls_mod = Modality( + id="HLS", + all_bands=("B", "G", "R", "N", "SW1", "SW2", "Fmask"), + stack_bands=("B", "G", "R", "N", "SW1", "SW2"), + chip_bands=("BANDS", "EVENT", "MASK", "Fmask") + ) + + modalities = [hls_mod, opera_rtc_mod] + + main(modalities) diff --git a/src/satchip/hls.py b/src/satchip/hls.py new file mode 100644 index 0000000..1f7022b --- /dev/null +++ b/src/satchip/hls.py @@ -0,0 +1,170 @@ +from datetime import timedelta, datetime +from pathlib import Path + +import earthaccess +from earthaccess.results import DataGranule +import numpy as np +import rasterio + + +def search_hls_data(start_date: datetime, bounding_box: tuple[float, float, float, float]) -> list[DataGranule]: + final_date = start_date + timedelta(days=1) + + # collection_ids = ["C2021957295-LPCLOUD"] # S2 + collection_ids = ["C2021957657-LPCLOUD", "C2021957295-LPCLOUD"] # S2, L30 + + results = earthaccess.search_data( + concept_id=collection_ids, + temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + bounding_box=bounding_box, + cloud_hosted=True, + ) + + return results + + +def band_from_hls_filename(filename): + # HLS.L30.T15TUG.2017167T165321.v2.0.B06.tif + parts = filename.split('.') + + sensor, band = parts[1], parts[-2] + + bands = { + "L30": { + "B02": "B", + "B03": "G", + "B04": "R", + "B05": "N", + "B06": "SW1", + "B07": "SW2", + "Fmask": "Fmask", + }, + "S30": { + "B02": "B", + "B03": "G", + "B04": "R", + "B08": "N", + "B11": "SW1", + "B12": "SW2", + "Fmask": "Fmask", + } + } + + try: + return bands[sensor][band] + except KeyError: + return '' + + +def make_merged_hls_name(template_filename: str) -> str: + parts = template_filename.split(".") + parts[4] = parts[4][0:7] + parts.pop(3) + parts[-2] = band_from_hls_filename(template_filename) + f_template_merge = ".".join(parts) + return f_template_merge + + +def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: + fmask_clear = np.array([ + 0, 4, 16, 20, 32, 36, 48, 52, + 64, 68, 80, 84, 96, 100, 112, 116, + 128, 132, 144, 148, 160, 164, 176, 180, + 192, 196, 208, 212, 224, 228, 240, 244 + ], dtype=Fmask.dtype) + + cloudmask = np.ones_like(Fmask, dtype=np.uint8) + cloudmask[np.isin(Fmask, fmask_clear)] = 0 + cloudmask[Fmask == 255] = 255 + + return cloudmask + + +def is_valid_hls(fmask_path: Path, event_path: Path): + with rasterio.open(fmask_path) as ds: + qc = clear_px_Fmask(ds.read(1)) + qc_profile = ds.profile + + with rasterio.open(event_path) as ds: + event_mask = ds.read(1) + event_profile = ds.profile + + print(qc_profile, event_profile) + + ny, nx = np.shape(qc) + mask = np.zeros((ny, nx), "uint8") + + ok = np.where((event_mask == 2) & (qc == 0)) + n_cf_event = len(ok[0]) + mask[ok] = 1 + + ok = np.where((event_mask == 1) & (qc != 255)) + n_valid_event = len(ok[0]) + + ok = np.where((event_mask == 1)) + n_event = len(ok[0]) + + pct_cf_event = 0 + if n_valid_event == 0: + print("No coverage") + else: + pct_cf_event = 100.0 * (n_cf_event / n_event) + print("Percent CF/valid in Event:", pct_cf_event) + + return pct_cf_event > 50 + + +def filter_hls_chips(chips: dict[str, dict]) -> list[dict]: + good_chips = [] + + for tile_id, chip in chips.items(): + with rasterio.open(chip["Fmask"]) as ds: + qc = clear_px_Fmask(ds.read(1)) + + with rasterio.open(chip["EVENT"]) as ds: + event = ds.read(1) + + ny, nx = qc.shape + n_px = 1.0 * ny * nx + + # cloud-free pixels (0 clear, 1 cloud, 255 nodata) + n_cf = len(np.where(qc == 0)[0]) + pct_cf = 100.0 * (n_cf / n_px) + + # event pixels + n_ev = len(np.where(event > 0)[0]) + + # pct of chip in event + pct_ev = 100.0 * (n_ev / n_px) if n_ev > 0 else 0 + + if pct_cf > 95 and pct_ev > 1: + good_chips.append(chip) + + return good_chips + + +def bytescale(arr, cmin=0, cmax=1, low=0, high=255): + # clip the data to be in the range of cmin to cmax + arr = np.clip(arr, cmin, cmax) + high = float(high) + low = float(low) + cmax = float(cmax) + cmin = float(cmin) + m = (high - low) / (cmax - cmin) # slope + b = high - (m * cmax) # intercept + arr = np.uint8((m * arr) + b) + return arr + + +def get_hls_img(hls_data: np.ndarray) -> np.ndarray: + # B04 + r = bytescale(np.sqrt(np.clip(hls_data[2] / 10000.0, 0, 2)), 0, 0.5) + + # B03 + g = bytescale(np.sqrt(np.clip(hls_data[1] / 10000.0, 0, 2)), 0, 0.5) + + # B02 + b = bytescale(np.sqrt(np.clip(hls_data[0] / 10000.0, 0, 2)), 0, 0.5) + + rgb = np.dstack((r, g, b)) + return rgb diff --git a/src/satchip/modality.py b/src/satchip/modality.py new file mode 100644 index 0000000..606b3d8 --- /dev/null +++ b/src/satchip/modality.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class Modality: + id: str + all_bands: Tuple[str, ...] + stack_bands: Tuple[str, ...] + chip_bands: Tuple[str, ...] diff --git a/src/satchip/mosaic.py b/src/satchip/mosaic.py new file mode 100644 index 0000000..b14e3fd --- /dev/null +++ b/src/satchip/mosaic.py @@ -0,0 +1,227 @@ +from pathlib import Path +import datetime + +import earthaccess +import numpy as np +import rasterio +from pyproj import Transformer +from rasterio.crs import CRS +from rasterio.merge import merge +from rasterio.warp import calculate_default_transform, reproject +from rasterio.mask import mask +from shapely.geometry import box + +from modality import Modality +import hls +import opera_rtc + + +def data_over_swath(swath, modalities: list[Modality], output_path: Path): + data_paths = { + "RAW": output_path / "RAW", + "REPROJECTED": output_path / "REPROJECTED", + } + + for item in output_path.glob('*.tif'): + if item.is_dir(): + continue + + item.unlink() + + for p in data_paths.values(): + p.mkdir(parents=True, exist_ok=True) + + swathID = f"{int(swath['swathID']):04d}" + + merged = {} + + for modality in modalities: + print(f'Localizing data for {modality.id}.') + + bounding_box = swath["buffered_event_background"].bounds + # transformer = Transformer.from_crs(32615, 4326, always_xy=True) + # minx, miny = transformer.transform(minx, miny) + # maxx, maxy = transformer.transform(maxx, maxy) + + start_date = swath["ls5hlsDate"] if modality.id == 'HLS' else swath["s1Date"] + print(start_date) + + results = search_data(bounding_box, start_date, modality) + + local_files = earthaccess.download( + results, local_path=data_paths["RAW"], show_progress=True + ) + + data_tifs = [f for f in local_files if f.name.endswith(".tif")] + reprojected_tifs = _reproject_files(data_tifs, output_path=data_paths["REPROJECTED"]) + + if len(data_tifs) == 0: + print(f"Skipping: no data for swath {swathID}") + continue + + mod_merged = {} + + for band in modality.all_bands: + band_files = [f for f in reprojected_tifs if band in band_from_filename(f.name, modality)] + merged_name = make_merge_name(swathID, start_date, band, modality) + + merged_band_path = _merge( + band_files, output_file=output_path / merged_name + ) + + print(f'Clipping band {band} data to same area') + clipped_band_path = _clip_over_swath(merged_band_path, swath) + mod_merged[band] = clipped_band_path + + print(f'Stacking bands for {modality.id}') + stacked_data = _stack_bands(mod_merged, data_bands=modality.stack_bands) + + merged[modality.id] = stacked_data + + breakpoint() + return merged + + +def search_data(bounding_box: tuple, start_date: datetime.datetime, modality: Modality): + results = {} + + if modality.id == 'HLS': + results = hls.search_hls_data(start_date=start_date, bounding_box=bounding_box) + elif modality.id == 'RTC': + results = opera_rtc.search_rtc_data(start_date=start_date, bounding_box=bounding_box) + + return results + + +def band_from_filename(filename, modality): + if modality.id == 'HLS': + band = hls.band_from_hls_filename(filename) + elif modality.id == 'RTC': + band = opera_rtc.band_from_rtc_filename(filename) + + return band + + +def make_merge_name(swathID: str, start_date: datetime.datetime, band: str, modality: Modality): + date_str = start_date.date().isoformat() + + return f'{swathID}.{modality.id}.{date_str}.{band}.tif' + + +def _reproject_files( + files: list[Path], output_path: Path +) -> list[Path]: + + reprojected_paths = [ + output_path / f"{granule.name}" for granule in files + ] + + for granule, output_path in zip(files, reprojected_paths): + if output_path.exists(): + continue + + print(f"reprojecting to wgs84: {output_path.name}") + _reproject_file(granule, output_path) + + return reprojected_paths + + +def _reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: + # https://rasterio.readthedocs.io/en/stable/topics/reproject.html#reprojecting-a-geotiff-dataset + with rasterio.open(local_file) as src: + dst_crs = CRS.from_epsg(epsg) + transform, width, height = calculate_default_transform( + src.crs, dst_crs, src.width, src.height, *src.bounds + ) + + dst_kwargs = src.meta.copy() + dst_kwargs.update( + {"crs": dst_crs, "transform": transform, "width": width, "height": height} + ) + + with rasterio.open(reprojected_file, "w", **dst_kwargs) as dst: + for i in range(1, src.count + 1): + reproject( + source=rasterio.band(src, i), + destination=rasterio.band(dst, i), + src_transform=src.transform, + src_crs=src.crs, + dst_transform=transform, + dst_crs=dst_crs, + ) + + +def _merge(band_files: list[Path], output_file: Path) -> Path: + band_datasets = [rasterio.open(rtc_tif) for rtc_tif in band_files] + + master_crs = band_datasets[0].crs + for ds in band_datasets[1:]: + if ds.crs != master_crs: + ds.crs = master_crs + + try: + mosaic, out_trans = merge(band_datasets) + mosaic = np.squeeze(mosaic) + + out_meta = band_datasets[0].meta.copy() + + out_meta.update( + { + "driver": "GTiff", + "height": mosaic.shape[0], + "width": mosaic.shape[1], + "transform": out_trans, + "crs": band_datasets[0].crs, + } + ) + + with rasterio.open(output_file, "w", **out_meta) as dst: + dst.write(mosaic, 1) + finally: + for ds in band_datasets: + ds.close() + + return output_file + + +def _stack_bands(merged: dict[str, Path], data_bands: tuple[str]) -> None: + with rasterio.open(merged[data_bands[0]]) as src: + meta = src.meta.copy() + + band = data_bands[0] + meta.update(count=len(data_bands), dtype=np.float32) + stacked_file_name = _rename(merged[band], f"{band}.tif", "BANDS.tif") + + with rasterio.open(stacked_file_name, "w", **meta) as dst: + for idx, band in enumerate(data_bands, start=1): + with rasterio.open(merged[band]) as src: + + dst.write(src.read(1), idx) + + merged["BANDS"] = stacked_file_name + return merged + + +def _rename(path: Path, extension: str, mask_name: str) -> Path: + return path.parent / path.name.replace(extension, mask_name) + + +def _clip_over_swath(input_path: Path, swath) -> Path: + bounding_box = swath["buffered_event_background"].bounds + + with rasterio.open(input_path) as src: + out_image, out_transform = mask(src, shapes=[box(*bounding_box)], crop=True) + out_meta = src.meta.copy() + + out_meta.update({ + "driver": "GTiff", + "height": out_image.shape[1], + "width": out_image.shape[2], + "transform": out_transform, + "crs": src.crs + }) + + with rasterio.open(input_path, "w", **out_meta) as dest: + dest.write(out_image) + + return input_path diff --git a/src/satchip/chip_data.py b/src/satchip/old/chip_data.py similarity index 75% rename from src/satchip/chip_data.py rename to src/satchip/old/chip_data.py index 847e82d..496d7d6 100644 --- a/src/satchip/chip_data.py +++ b/src/satchip/old/chip_data.py @@ -1,8 +1,10 @@ import argparse +import glob from collections import Counter from datetime import datetime from pathlib import Path +import earthaccess import numpy as np import xarray as xr from shapely.geometry import box @@ -74,8 +76,7 @@ def chip_data( def create_chips( label_paths: list[Path], platform: str, - date_start: datetime, - date_end: datetime, + dates: utils.DateRange | list[datetime], strategy: str, max_cloud_pct: int, chip_dir: Path, @@ -84,19 +85,24 @@ def create_chips( platform_dir = chip_dir / platform platform_dir.mkdir(parents=True, exist_ok=True) - opts: utils.ChipDataOpts = {'strategy': strategy, 'date_start': date_start, 'date_end': date_end} + opts: utils.ChipDataOpts = {'strategy': strategy, 'dates': dates} if platform in ['S2L2A', 'HLS']: opts['max_cloud_pct'] = max_cloud_pct + if platform in ['S1RTC', 'HLS']: + earthaccess.login() + chips = get_chips(label_paths) chip_names = [c.name for c in chips] if len(chip_names) != len(set(chip_names)): duplicates = [name for name, count in Counter(chip_names).items() if count > 1] msg = f'Duplicate sample locations not supported. Duplicate chips: {", ".join(duplicates)}' raise NotImplementedError(msg) + chip_paths = [ platform_dir / (x.with_suffix('').with_suffix('').name + f'_{platform}.zarr.zip') for x in label_paths ] + if platform == 'HYP3S1RTC': rtc_paths_for_chips = get_rtc_paths_for_chips(chips, image_dir, opts) opts['local_hyp3_paths'] = rtc_paths_for_chips @@ -109,11 +115,26 @@ def create_chips( def main() -> None: parser = argparse.ArgumentParser(description='Chip a label image') - parser.add_argument('labelpath', type=Path, help='Path to the label directory') + + parser.add_argument('labels', type=str, help='Path or Glob pattern for label chips') + parser.add_argument( 'platform', choices=['S1RTC', 'S2L2A', 'HLS', 'HYP3S1RTC'], type=str, help='Dataset to create chips for' ) - parser.add_argument('daterange', type=str, help='Inclusive date range to search for data in the format Ymd-Ymd') + + date_group = parser.add_mutually_exclusive_group(required=True) + + date_group.add_argument( + '--daterange', + nargs=2, + type=str, + metavar=('START', 'END'), + help='Inclusive date range in the format YYYY-mm-dd YYYY-mm-dd', + ) + date_group.add_argument( + '--dates', nargs='+', type=str, help='Space-separated list of specific dates in format YYYY-mm-dd' + ) + parser.add_argument('--maxcloudpct', default=100, type=int, help='Maximum percent cloud cover for a data chip') parser.add_argument('--chipdir', default='.', type=Path, help='Output directory for the chips') parser.add_argument( @@ -122,24 +143,37 @@ def main() -> None: parser.add_argument( '--strategy', default='BEST', - choices=['BEST', 'ALL'], + choices=['BEST', 'ALL', 'SPECIFIC'], type=str, help='Strategy to use when multiple scenes are found (default: BEST)', ) args = parser.parse_args() + args.platform = args.platform.upper() assert 0 <= args.maxcloudpct <= 100, 'maxcloudpct must be between 0 and 100' - date_start, date_end = [datetime.strptime(d, '%Y%m%d') for d in args.daterange.split('-')] - assert date_start < date_end, 'start date must be before end date' - label_paths = list(args.labelpath.glob('*.zarr.zip')) - assert len(label_paths) > 0, f'No label files found in {args.labelpath}' + + dates: utils.DateRange | list[datetime] + if args.daterange: + start_date = datetime.strptime(args.daterange[0], '%Y-%m-%d') + end_date = datetime.strptime(args.daterange[1], '%Y-%m-%d') + assert start_date < end_date, 'start date must be before end date' + + dates = utils.DateRange(start_date, end_date) + else: + dates = [datetime.strptime(d, '%Y-%m-%d') for d in args.dates] + args.strategy = 'SPECIFIC' + + if '*' in args.labels or '?' in args.labels or '[' in args.labels: + label_paths = [Path(p) for p in glob.glob(args.labels)] + else: + label_paths = list(Path(args.labels).glob('*.zarr.zip')) + + assert len(label_paths) > 0, f'No label files found in {args.labels}' if args.imagedir is None: args.imagedir = args.chipdir / 'IMAGES' - create_chips( - label_paths, args.platform, date_start, date_end, args.strategy, args.maxcloudpct, args.chipdir, args.imagedir - ) + create_chips(label_paths, args.platform, dates, args.strategy, args.maxcloudpct, args.chipdir, args.imagedir) if __name__ == '__main__': diff --git a/src/satchip/chip_hls.py b/src/satchip/old/chip_hls.py similarity index 84% rename from src/satchip/chip_hls.py rename to src/satchip/old/chip_hls.py index 4452356..b6fb9b8 100644 --- a/src/satchip/chip_hls.py +++ b/src/satchip/old/chip_hls.py @@ -15,6 +15,8 @@ from satchip.terra_mind_grid import TerraMindChip +earthaccess.login() + HLS_L_BANDS = OrderedDict( { 'B01': 'COASTAL', @@ -78,19 +80,25 @@ def get_scenes( Returns: The best HLS items. """ - assert strategy in ['BEST', 'ALL'], 'Strategy must be either BEST or ALL' + assert strategy in ['BEST', 'ALL', 'SPECIFIC'], 'Strategy must be either BEST or ALL' overlapping_items = [x for x in items if get_pct_intersect(x['umm'], roi) > 95] best_first = sorted(overlapping_items, key=lambda x: (-get_pct_intersect(x['umm'], roi), get_date(x['umm']))) + valid_scenes = [] + for item in best_first: product_id = get_product_id(item['umm']) n_products = len(list(image_dir.glob(f'{product_id}*'))) + if n_products < 15: earthaccess.download([item], image_dir, pqdm_kwargs={'disable': True}) + fmask_path = image_dir / f'{product_id}.v2.0.Fmask.tif' assert fmask_path.exists(), f'File not found: {fmask_path}' + qual_da = rioxarray.open_rasterio(fmask_path).rio.clip_box(*roi.bounds, crs='EPSG:4326') # type: ignore bit_masks = np.unpackbits(qual_da.data[0][..., np.newaxis], axis=-1) + # Looks for a 1 in the 4th, 6th and 7th bit of the Fmask (reverse order). See table 9 and appendix A of: # https://lpdaac.usgs.gov/documents/1698/HLS_User_Guide_V2.pdf bad_pixels = (bit_masks[..., 4] == 1) | (bit_masks[..., 6] == 1) | (bit_masks[..., 7] == 1) @@ -104,22 +112,38 @@ def get_scenes( return valid_scenes +def search_for_data(dates: utils.DateRange | list[datetime], bounds: utils.Bounds) -> list: + results = [] + if isinstance(dates, utils.DateRange): + results = earthaccess.search_data( + short_name=['HLSL30', 'HLSS30'], bounding_box=bounds, temporal=(dates.start, dates.end + timedelta(days=1)) + ) + else: + for date in dates: + day_results = earthaccess.search_data( + short_name=['HLSL30', 'HLSS30'], bounding_box=bounds, temporal=(date, date + timedelta(days=1)) + ) + results.extend(day_results) + + return results + + def get_hls_data(chip: TerraMindChip, image_dir: Path, opts: utils.ChipDataOpts) -> xr.Dataset: """Returns XArray DataArray of a Harmonized Landsat Sentinel-2 image for the given bounds and closest collection after date. """ - date_start = opts['date_start'] - date_end = opts['date_end'] + timedelta(days=1) # inclusive end - earthaccess.login() - results = earthaccess.search_data( - short_name=['HLSL30', 'HLSS30'], bounding_box=chip.bounds, temporal=(date_start, date_end) - ) - assert len(results) > 0, f'No HLS scenes found for chip {chip.name} between {date_start} and {date_end}.' + dates = opts['dates'] + + results = search_for_data(dates, utils.Bounds(*chip.bounds)) + assert len(results) > 0, f'No HLS scenes found for chip {chip.name} {utils.dates_error_msg(dates)}.' + roi = shapely.box(*chip.bounds) roi_buffered = roi.buffer(0.01) max_cloud_pct = opts.get('max_cloud_pct', 100) strategy = opts.get('strategy', 'BEST').upper() - timesteps = get_scenes(results, roi, max_cloud_pct, strategy, image_dir) + + timesteps = get_scenes(results, roi_buffered, max_cloud_pct, strategy, image_dir) + template = create_template_da(chip) timestep_arrays = [] for scene in timesteps: diff --git a/src/satchip/chip_hyp3s1rtc.py b/src/satchip/old/chip_hyp3s1rtc.py similarity index 100% rename from src/satchip/chip_hyp3s1rtc.py rename to src/satchip/old/chip_hyp3s1rtc.py diff --git a/src/satchip/chip_label.py b/src/satchip/old/chip_label.py similarity index 100% rename from src/satchip/chip_label.py rename to src/satchip/old/chip_label.py diff --git a/src/satchip/chip_operas1rtc.py b/src/satchip/old/chip_operas1rtc.py similarity index 80% rename from src/satchip/chip_operas1rtc.py rename to src/satchip/old/chip_operas1rtc.py index 5811998..055232e 100644 --- a/src/satchip/chip_operas1rtc.py +++ b/src/satchip/old/chip_operas1rtc.py @@ -87,25 +87,54 @@ def get_scenes(groups: list[RTCGroup], roi: shapely.geometry.Polygon, strategy: return intersecting[:1] elif strategy == 'ALL': return intersecting + elif strategy == 'SPECIFIC': + return intersecting else: raise ValueError(f'Strategy must be either BEST or ALL. Got {strategy}') +@utils.retry_on_connection_error(max_retries=5, backoff_factor=2) +def search_for_data(dates: utils.DateRange | list[datetime], bounds: utils.Bounds) -> list: + results = [] + + if isinstance(dates, utils.DateRange): + results = earthaccess.search_data( + short_name=['OPERA_L2_RTC-S1_V1'], + bounding_box=bounds, + temporal=(dates.start, dates.end + timedelta(days=1)), + ) + else: + for date in dates: + day_results = earthaccess.search_data( + short_name=['OPERA_L2_RTC-S1_V1'], bounding_box=bounds, temporal=(date, date + timedelta(days=1)) + ) + results.extend(day_results) + + return results + + def get_operartc_data(chip: TerraMindChip, image_dir: Path, opts: utils.ChipDataOpts) -> xr.Dataset: """Returns XArray DataArray of a OPERA S1-RTC for the given chip and selection startegy.""" - date_start = opts['date_start'] - date_end = opts['date_end'] + timedelta(days=1) # inclusive end - earthaccess.login() - results = earthaccess.search_data( - short_name=['OPERA_L2_RTC-S1_V1'], bounding_box=chip.bounds, temporal=(date_start, date_end) - ) - results = filter_to_dualpol(results) - rtc_groups = group_rtcs(results) + + dates = opts['dates'] + roi = shapely.box(*chip.bounds) roi_buffered = roi.buffer(0.01) + + results = search_for_data(dates, utils.Bounds(*roi_buffered.bounds)) + dualpol = filter_to_dualpol(results) + + rtc_groups = group_rtcs(dualpol) strategy = opts.get('strategy', 'BEST').upper() timesteps = get_scenes(rtc_groups, roi_buffered, strategy) - assert len(timesteps) > 0, f'No OPERA RTC scenes found for chip {chip.name} between {date_start} and {date_end}.' + + assert len(timesteps) > 0, f'No OPERA RTC scenes found for chip {chip.name} {utils.dates_error_msg(dates)}' + + if isinstance(dates, list): + print(dates, timesteps) + breakpoint() + assert len(timesteps) == len(dates) + vrts = [timestep.download(image_dir) for timestep in timesteps] template = create_template_da(chip) timestep_arrays = [] diff --git a/src/satchip/chip_sentinel2.py b/src/satchip/old/chip_sentinel2.py similarity index 85% rename from src/satchip/chip_sentinel2.py rename to src/satchip/old/chip_sentinel2.py index f4725b4..34ae90c 100644 --- a/src/satchip/chip_sentinel2.py +++ b/src/satchip/old/chip_sentinel2.py @@ -91,11 +91,11 @@ def get_scenes( The best Sentinel-2 L2A item. """ strategy = strategy.upper() - assert strategy in ['BEST', 'ALL'], 'Strategy must be either BEST or ALL' assert len(items) > 0, 'No Sentinel-2 L2A scenes found for chip.' items = [item for item in items if get_pct_intersect(item.geometry, roi) > 0.95] best_first = sorted(items, key=lambda x: (-get_pct_intersect(x.geometry, roi), x.datetime)) valid_scenes = [] + for item in best_first: scl_href = item.assets['scl'].href local_path = fetch_s3_file(scl_href, image_dir) @@ -131,6 +131,36 @@ def get_s2_version(item: Item) -> int: return latest_items +def search_for_data(dates: utils.DateRange | list[datetime], roi: shapely.Polygon) -> list: + results = [] + client = Client.open('https://earth-search.aws.element84.com/v1') + + if isinstance(dates, utils.DateRange): + date_end = dates.end + timedelta(days=1) + date_range = f'{datetime.strftime(dates.start, "%Y-%m-%d")}/{datetime.strftime(date_end, "%Y-%m-%d")}' + search = client.search( + collections=['sentinel-2-l2a'], + intersects=roi, + datetime=date_range, + max_items=1000, + ) + + results = list(search.item_collection()) + else: + for date in dates: + search = client.search( + collections=['sentinel-2-l2a'], + intersects=roi, + datetime=datetime.strftime(date, '%Y-%m-%d'), + max_items=1000, + ) + results.extend(search.item_collection()) + + results = get_latest_image_versions(results) + + return results + + def get_s2l2a_data(chip: TerraMindChip, image_dir: Path, opts: utils.ChipDataOpts) -> xr.Dataset: """Get XArray DataArray of Sentinel-2 L2A image for the given bounds and best collection parameters. @@ -146,28 +176,20 @@ def get_s2l2a_data(chip: TerraMindChip, image_dir: Path, opts: utils.ChipDataOpt Returns: XArray Dataset containing the Sentinel-2 L2A image data. """ - date_start = opts['date_start'] - date_end = opts['date_end'] + timedelta(days=1) # inclusive end - date_range = f'{datetime.strftime(date_start, "%Y-%m-%d")}/{datetime.strftime(date_end, "%Y-%m-%d")}' + dates = opts['dates'] + roi = shapely.box(*chip.bounds) roi_buffered = roi.buffer(0.01) - client = Client.open('https://earth-search.aws.element84.com/v1') - search = client.search( - collections=['sentinel-2-l2a'], - intersects=roi, - datetime=date_range, - max_items=1000, - ) - assert len(search.item_collection()) > 0, ( - f'No Sentinel-2 L2A scenes found for chip {chip.name} between {date_start} and {date_end}.' - ) - assert len(search.item_collection()) < 1000, ( - 'Too many Sentinel-2 L2A scenes found for chip. Please narrow the date range.' - ) - items = list(search.item_collection()) - items = get_latest_image_versions(items) + + items = search_for_data(dates, roi) max_cloud_pct = opts.get('max_cloud_pct', 100) strategy = opts.get('strategy', 'BEST') + + assert len(items) > 0, ( + f'No Sentinel-2 L2A scenes found for chip {chip.name} between {utils.dates_error_msg(dates)}.' + ) + assert len(items) < 1000, 'Too many Sentinel-2 L2A scenes found for chip. Please narrow the date range.' + timesteps = get_scenes(items, roi, strategy, max_cloud_pct, image_dir) urls = [item.assets[band.lower()].href for item in timesteps for band in S2_BANDS.values()] diff --git a/src/satchip/chip_view.py b/src/satchip/old/chip_view.py similarity index 100% rename from src/satchip/chip_view.py rename to src/satchip/old/chip_view.py diff --git a/src/satchip/chip_xr_base.py b/src/satchip/old/chip_xr_base.py similarity index 100% rename from src/satchip/chip_xr_base.py rename to src/satchip/old/chip_xr_base.py diff --git a/src/satchip/major_tom_grid.py b/src/satchip/old/major_tom_grid.py similarity index 100% rename from src/satchip/major_tom_grid.py rename to src/satchip/old/major_tom_grid.py diff --git a/src/satchip/terra_mind_grid.py b/src/satchip/old/terra_mind_grid.py similarity index 100% rename from src/satchip/terra_mind_grid.py rename to src/satchip/old/terra_mind_grid.py diff --git a/src/satchip/old/utils.py b/src/satchip/old/utils.py new file mode 100644 index 0000000..e4ffd10 --- /dev/null +++ b/src/satchip/old/utils.py @@ -0,0 +1,116 @@ +import datetime +import functools +import time +import warnings +from collections.abc import Callable +from pathlib import Path +from typing import NamedTuple, ParamSpec, TypeVar, TypedDict + +import xarray as xr +import zarr +from pyproj import CRS, Transformer +from requests.exceptions import ConnectionError + + +class RtcImageSet(TypedDict): + VV: Path + VH: Path + + +class Bounds(NamedTuple): + minx: float + miny: float + maxx: float + maxy: float + + +class DateRange(NamedTuple): + start: datetime.datetime + end: datetime.datetime + + +class ChipDataRequiredOpts(TypedDict): + strategy: str + dates: DateRange | list[datetime.datetime] + + +class ChipDataOpts(ChipDataRequiredOpts, total=False): + max_cloud_pct: int + local_hyp3_paths: dict[str, list[RtcImageSet]] + + +def dates_error_msg(dates: DateRange | list[datetime.datetime]) -> str: + return f'between {dates.start} and {dates.end}' if isinstance(dates, DateRange) else f'for dates {dates}' + + +def get_overall_bounds(bounds: list) -> Bounds: + minx = min([b[0] for b in bounds]) + miny = min([b[1] for b in bounds]) + maxx = max([b[2] for b in bounds]) + maxy = max([b[3] for b in bounds]) + return Bounds(minx, miny, maxx, maxy) + + +def get_epsg4326_point(x: float, y: float, in_epsg: int) -> tuple[float, float]: + if in_epsg == 4326: + return x, y + in_crs = CRS.from_epsg(in_epsg) + out_crs = CRS.from_epsg(4326) + transformer = Transformer.from_crs(in_crs, out_crs, always_xy=True) + newx, newy = transformer.transform(x, y) + return round(newx, 5), round(newy, 5) + + +def get_epsg4326_bbox( + bounds: tuple[float, float, float, float], in_epsg: int, buffer: float = 0.1 +) -> tuple[float, float, float, float]: + minx, miny = get_epsg4326_point(bounds[0], bounds[1], in_epsg) + maxx, maxy = get_epsg4326_point(bounds[2], bounds[3], in_epsg) + bbox = minx - buffer, miny - buffer, maxx + buffer, maxy + buffer + return bbox + + +def save_chip(dataset: xr.Dataset, save_path: str | Path) -> None: + """Save a zipped zarr archive""" + store = zarr.storage.ZipStore(save_path, mode='w') + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', message='Duplicate name:', module='zipfile') + dataset.to_zarr(store) # type: ignore[call-overload] + store.close() + + +def load_chip(label_path: str | Path) -> xr.Dataset: + """Load a zipped zarr archive""" + store = zarr.storage.ZipStore(label_path, read_only=True) + dataset = xr.open_zarr(store) + return dataset + + +P = ParamSpec('P') +R = TypeVar('R') + + +def retry_on_connection_error( + max_retries: int = 3, + backoff_factor: float = 1 +) -> Callable[[Callable[P, R]], Callable[P, R]]: + + def decorator(func: Callable[P, R]) -> Callable[P, R]: + + @functools.wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + for attempt in range(max_retries): + try: + return func(*args, **kwargs) + except (ConnectionError, OSError) as e: + if attempt == max_retries - 1: + print(f"Failed after {max_retries} attempts: {e}") + raise e + + wait_time = backoff_factor * (2 ** attempt) + time.sleep(wait_time) + + raise RuntimeError("Unexpected exit from retry loop") + + return wrapper + return decorator diff --git a/src/satchip/opera_rtc.py b/src/satchip/opera_rtc.py new file mode 100644 index 0000000..cf54737 --- /dev/null +++ b/src/satchip/opera_rtc.py @@ -0,0 +1,106 @@ +from datetime import timedelta, datetime +from pathlib import Path + +import earthaccess +from earthaccess.results import DataGranule +import numpy as np +import rasterio + + +def search_rtc_data(start_date: datetime, bounding_box: tuple[float, float, float, float]) -> list[DataGranule]: + final_date = start_date + timedelta(days=1) + + results = earthaccess.search_data( + short_name=["OPERA_L2_RTC-S1_V1"], + temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + bounding_box=bounding_box, + ) + + return results + + +def band_from_rtc_filename(filename): + # OPERA_L2_RTC-S1_T063-133415-IW2_20170620T001327Z_20250925T045340Z_S1A_30_v1.0_VV.tif + return filename.split('_')[-1].split('.')[0] + + +def make_merged_rtc_name(template_filename: str) -> str: + """ + https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/#naming-convention + swathID.OPERA_L2_RTC-S1_[BurstID]_[StartDateTime]_[ProductGenerationDateTime] _[Sensor]_[PixelSpacing]_[ProductVersion]_[LayerName].Ext + + Input: 1442.OPERA_L2_RTC-S1_T063-133415-IW2_20170620T001327Z_20250925T045340Z_S1A_30_v1.0_VV.tif + Returns: 1442.OPERA_L2_RTC-133415-IW2_20170620_S1A_30_v1.0_VV.tif + """ + + # ['1442.OPERA', 'L2', 'RTC-S1', 'T063-133415-IW2', '20170620T001327Z', '20250925T045340Z', 'S1A', '30', 'v1.0', 'VV.tif'] + name_parts = template_filename.split("_") + + name_parts.pop(5) # Remove Product Generation Time + name_parts.pop(3) # Remove Burst ID + + return "_".join(name_parts) + + +def is_valid_rtc(mask_path: Path, label_path: Path) -> bool: + with rasterio.open(mask_path) as ds: + validity_mask = ds.read(1) + + with rasterio.open(label_path) as ds: + event_mask = ds.read(1) + + is_event_pixel = event_mask == 1 + # https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/#validity-mask + is_valid_pixel = np.isin(validity_mask, [0, 1]) + + total_event_pixels = is_event_pixel.sum() + valid_event_pixels = (is_event_pixel & is_valid_pixel).sum() + + pct_valid_data = 100.0 * valid_event_pixels / total_event_pixels + print(f"Percent of the event with valid data: {pct_valid_data:.1f}%") + + return pct_valid_data > 50.0 + + +def filter_rtc_chips(chips: dict[str, dict]) -> list[dict]: + good_chips = [] + + for tile_id, chip in chips.items(): + with rasterio.open(chip["BANDS"]) as ds: + rtc_data = ds.read() + + with rasterio.open(chip["EVENT"]) as ds: + event_mask = ds.read(1) + + has_nan_pixels = np.isnan(rtc_data).sum() > 0 + + num_pixels = event_mask.size + num_event_pixels = np.count_nonzero(event_mask > 0) + + pct_pixels_over_event = 100.0 * (num_event_pixels / num_pixels) + data_overlaps_event = pct_pixels_over_event > 1 + + if not has_nan_pixels and data_overlaps_event: + good_chips.append(chip) + + return good_chips + + +def normalize_image_array( + input_array: np.ndarray, vmin: float, vmax: float +) -> np.ndarray: + input_array = input_array.astype(float) + scaled_array = (input_array - vmin) / (vmax - vmin) + scaled_array[np.isnan(input_array)] = 0 + normalized_array = np.round(np.clip(scaled_array, 0, 1) * 255).astype(np.uint8) + + return normalized_array + + +def get_rtc_img(rtc_data: np.ndarray) -> np.ndarray: + vv = normalize_image_array(np.sqrt(rtc_data[0]), 0.14, 0.52) + vh = normalize_image_array(np.sqrt(rtc_data[1]), 0.05, 0.259) + + img = np.stack([vv, vh, vv], axis=-1) + + return img diff --git a/tests/test_chip_hyp3s1rtc.py b/tests/test_chip_hyp3s1rtc.py deleted file mode 100644 index 0216cab..0000000 --- a/tests/test_chip_hyp3s1rtc.py +++ /dev/null @@ -1,172 +0,0 @@ -import datetime -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -from shapely.geometry import box, mapping - -from satchip import chip_hyp3s1rtc, utils - - -def test_bounds_check(): - chip_hyp3s1rtc._check_bounds_size(utils.Bounds(0, 0, 1, 1)) - chip_hyp3s1rtc._check_bounds_size(utils.Bounds(0, 0, 2.9, 1)) - chip_hyp3s1rtc._check_bounds_size(utils.Bounds(-107.79192, 45.74287, -105.01543, 46.48598)) - - with pytest.raises(AssertionError): - chip_hyp3s1rtc._check_bounds_size(utils.Bounds(0, 0, 3, 1)) - - -def test_get_granules(): - bounds = utils.Bounds(-107.79192, 45.74287, -105.01543, 46.48598) - date_start = datetime.datetime(2020, 7, 7) - date_end = date_start + datetime.timedelta(days=14) - - mock_search_result = ['granule1', 'granule2'] - - with patch('satchip.chip_hyp3s1rtc.asf.geo_search', return_value=mock_search_result) as mock_geo_search: - results = chip_hyp3s1rtc._get_granules(bounds, date_start, date_end) - - mock_geo_search.assert_called_once() - - assert results == mock_search_result - - args, kwargs = mock_geo_search.call_args - assert ( - kwargs['intersectsWith'] - == 'POLYGON ((-105.01543 45.74287, -105.01543 46.48598, -107.79192 46.48598, -107.79192 45.74287, -105.01543 45.74287))' - ) - assert kwargs['start'] == date_start - assert kwargs['end'] == date_end + datetime.timedelta(days=1) - - -def test_get_slcs_for_each_chip_custom_intersect(): - granule1 = MagicMock() - granule1.geometry = mapping(box(0, 0, 2, 2)) - granule1.properties = {'startTime': '2025-01-01T00:00:00Z'} - - granule2 = MagicMock() - granule2.geometry = mapping(box(3, 3, 5, 5)) - granule2.properties = {'startTime': '2025-01-02T00:00:00Z'} - - granule3 = MagicMock() - granule3.geometry = mapping(box(10, 10, 15, 15)) - granule3.properties = {'startTime': '2025-01-03T00:00:00Z'} - - chip1 = MagicMock() - chip1.name = 'chip1' - chip1.bounds = [0, 0, 1, 1] - - chip2 = MagicMock() - chip2.name = 'chip2' - chip2.bounds = [1, 1, 2, 2] - - chip3 = MagicMock() - chip3.name = 'chip3' - chip3.bounds = [3, 3, 4, 4] - - chips = [chip1, chip2, chip3] - granules = [granule1, granule2, granule3] - - result = chip_hyp3s1rtc._get_slcs_for_each_chip(chips, granules, strategy='BEST') # type: ignore - - assert result['chip1'] == [granule1] - assert result['chip2'] == [granule1] - assert result['chip3'] == [granule2] - - -def test_get_slcs_for_each_chip_with_different_strategies(): - granule1 = MagicMock() - granule1.geometry = mapping(box(0, 0, 1, 1)) - granule1.properties = {'startTime': '2025-01-01T00:00:00Z'} - - granule2 = MagicMock() - granule2.geometry = mapping(box(0, 0, 5, 5)) - granule2.properties = {'startTime': '2025-01-02T00:00:00Z'} - - granule3 = MagicMock() - granule3.geometry = mapping(box(0, 0, 15, 15)) - granule3.properties = {'startTime': '2025-01-03T00:00:00Z'} - - chip1 = MagicMock() - chip1.name = 'chip1' - chip1.bounds = [0, 0, 5, 10] - - chips = [chip1] - granules = [granule1, granule2, granule3] - - result = chip_hyp3s1rtc._get_slcs_for_each_chip(chips, granules, strategy='BEST', intersection_pct=49) # type: ignore - assert result['chip1'] == [granule3] - - result = chip_hyp3s1rtc._get_slcs_for_each_chip(chips, granules, strategy='ALL', intersection_pct=49) # type: ignore - assert result['chip1'] == [granule3, granule2] - - -def test_get_slcs_for_each_chip_no_matches(): - chip = MagicMock() - chip.name = 'chip1' - chip.bounds = [0, 0, 1, 1] - - with pytest.raises(ValueError, match='No products found for chip chip1'): - chip_hyp3s1rtc._get_slcs_for_each_chip([chip], [], strategy='BEST') - - -class MockS1Product: - def __init__(self, scene_name: str): - self.properties = {'sceneName': scene_name} - - -def test_get_rtcs_for(): - slcs_for_chips = { - 'chip_001': [MockS1Product('SLC_1'), MockS1Product('SLC_2')], - 'chip_002': [MockS1Product('SLC_3'), MockS1Product('SLC_4')], - } - scratch_dir = Path('/tmp') - - mock_jobs = [] - for slc_name in ['SLC_1', 'SLC_2', 'SLC_3', 'SLC_4']: - job = MagicMock() - job.job_parameters = {'granules': [slc_name]} - mock_jobs.append(job) - - with ( - patch('satchip.chip_hyp3s1rtc._process_rtcs', return_value=mock_jobs) as mock_process_rtcs, - patch('satchip.chip_hyp3s1rtc._download_hyp3_rtc') as mock_download, - ): - - def mock_download_fn(job, scratch): - return { - 'VV': Path(f'/tmp/{job.job_parameters["granules"][0]}_rtc_VV.tif'), - 'VH': Path(f'/tmp/{job.job_parameters["granules"][0]}_rtc_VH.tif'), - } - - mock_download.side_effect = mock_download_fn - - result = chip_hyp3s1rtc._get_rtcs_for(slcs_for_chips, scratch_dir) - - expected = { - 'chip_001': [ - { - 'VV': Path('/tmp/SLC_1_rtc_VV.tif'), - 'VH': Path('/tmp/SLC_1_rtc_VH.tif'), - }, - { - 'VV': Path('/tmp/SLC_2_rtc_VV.tif'), - 'VH': Path('/tmp/SLC_2_rtc_VH.tif'), - }, - ], - 'chip_002': [ - { - 'VV': Path('/tmp/SLC_3_rtc_VV.tif'), - 'VH': Path('/tmp/SLC_3_rtc_VH.tif'), - }, - { - 'VV': Path('/tmp/SLC_4_rtc_VV.tif'), - 'VH': Path('/tmp/SLC_4_rtc_VH.tif'), - }, - ], - } - - assert result == expected - mock_process_rtcs.assert_called_once_with({'SLC_1', 'SLC_2', 'SLC_3', 'SLC_4'}) - assert mock_download.call_count == 4 diff --git a/tests/test_chip_sentinel2.py b/tests/test_chip_sentinel2.py deleted file mode 100644 index 32a81d4..0000000 --- a/tests/test_chip_sentinel2.py +++ /dev/null @@ -1,24 +0,0 @@ -from collections import namedtuple - -from satchip.chip_sentinel2 import get_latest_image_versions - - -ItemStub = namedtuple('ItemStub', ['id', 'properties']) - - -def test_get_latest_image_versions(): - items = [ - ItemStub(id='S2B_13TEG_20190623_0_L2A', properties={'s2:sequence': 0}), - ItemStub(id='S2B_13TEG_20190623_1_L2A', properties={'s2:sequence': 1}), - ItemStub(id='S2A_13TEG_20190621_0_L2A', properties={'s2:sequence': 0}), - ItemStub(id='S2A_13TEG_20190618_0_L2A', properties={'s2:sequence': 0}), - ItemStub(id='S2A_13TEG_20190618_1_L2A', properties={'s2:sequence': 1}), - ItemStub(id='S2A_13TEG_20190618_3_L2A', properties={'s2:sequence': 2}), - ] - - latest_items = get_latest_image_versions(items) # type: ignore - - assert len(latest_items) == 3 - assert any(item.id == 'S2B_13TEG_20190623_1_L2A' for item in latest_items) - assert any(item.id == 'S2A_13TEG_20190621_0_L2A' for item in latest_items) - assert any(item.id == 'S2A_13TEG_20190618_3_L2A' for item in latest_items) diff --git a/tests/test_integration.py b/tests/test_integration.py deleted file mode 100644 index dba034c..0000000 --- a/tests/test_integration.py +++ /dev/null @@ -1,59 +0,0 @@ -from datetime import datetime -from pathlib import Path - -import pytest -from osgeo import gdal - -from satchip.chip_data import create_chips -from satchip.chip_label import chip_labels - - -gdal.UseExceptions() - - -def create_dataset(outpath: Path, start: tuple[int, int]) -> Path: - x, y = start - pixel_size = 10 - cols, rows = 512, 512 - driver = gdal.GetDriverByName('GTiff') - dataset = driver.Create(str(outpath), cols, rows, 1, gdal.GDT_UInt16) - dataset.SetGeoTransform((x, pixel_size, 0, y, 0, -pixel_size)) - dataset.SetProjection('EPSG:32611') - array = dataset.GetRasterBand(1).ReadAsArray() - array[:, :] = 0 - array[128:384, 128:384] = 1 - dataset.GetRasterBand(1).WriteArray(array) - dataset.FlushCache() - dataset = None - return outpath - - -def create_label_and_data(label_tif, out_dir, image_dir): - chip_labels(label_tif, datetime.fromisoformat('20240115'), out_dir) - for platform in ['S2L2A', 'HLS', 'S1RTC']: - create_chips( - list((out_dir / 'LABEL').glob('*.zarr.zip')), - platform, - datetime.fromisoformat('20240101'), - datetime.fromisoformat('20240215'), - 'BEST', - 20, - out_dir, - image_dir, - ) - - -@pytest.mark.integration -def test_integration(): - data_dir = Path('integration_test') - train_dir = data_dir / 'train' - train_dir.mkdir(parents=True, exist_ok=True) - val_dir = data_dir / 'val' - val_dir.mkdir(parents=True, exist_ok=True) - image_dir = data_dir / 'images' - image_dir.mkdir(parents=True, exist_ok=True) - - train_tif = create_dataset(data_dir / 'train.tif', (431795, 3943142)) - create_label_and_data(train_tif, train_dir, image_dir) - val_tif = create_dataset(data_dir / 'val.tif', (431795, 3943142 - 10 * 512)) - create_label_and_data(val_tif, val_dir, image_dir) diff --git a/tests/test_stub.py b/tests/test_stub.py deleted file mode 100644 index 31ba320..0000000 --- a/tests/test_stub.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_stub(): - assert True diff --git a/tests/test_terra_mind_grid.py b/tests/test_terra_mind_grid.py deleted file mode 100644 index 93b4d15..0000000 --- a/tests/test_terra_mind_grid.py +++ /dev/null @@ -1,23 +0,0 @@ -import numpy as np -import pytest - -from satchip.terra_mind_grid import TerraMindGrid - - -@pytest.mark.parametrize( - 'tm_name, point', - [ - ('374U_897R_3_2', [96.80658, 33.62825]), - ('735U_418L_0_1', [-92.35263, 66.07510]), - ('611U_462L_2_1', [-72.02323, 54.93515]), - ], -) -def test_terra_mind_grid(tm_name, point): - mt_name = f'{tm_name.split("_")[0]}_{tm_name.split("_")[1]}' - tmp_grid = TerraMindGrid((np.floor(point[1]), np.ceil(point[1])), (np.floor(point[0]), np.ceil(point[0]))) - # mt_chip = [x for x in tmp_grid.major_tom_chips if x.name == mt_name][0] - chips = [x for x in tmp_grid.terra_mind_chips if x.name.startswith(mt_name)] - in_lon = [x for x in chips if x.bounds[0] < point[0] < x.bounds[2]] - in_lon_lat = [x for x in in_lon if x.bounds[1] < point[1] < x.bounds[3]] - assert len(in_lon_lat) == 1 - assert in_lon_lat[0].name == tm_name From 50772d045ff9e972c2f1c0a5e9500619598e6c69 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 17 Mar 2026 15:46:50 -0800 Subject: [PATCH 02/21] Mosaic chips --- .gitignore | 1 + src/satchip/generate_chips.py | 193 ++++++++++++++++++---------------- src/satchip/mosaic.py | 130 +++++++++++++++++------ 3 files changed, 199 insertions(+), 125 deletions(-) diff --git a/.gitignore b/.gitignore index 4dc2ca6..00087c9 100644 --- a/.gitignore +++ b/.gitignore @@ -257,3 +257,4 @@ chips *.shx *.zip *.qgz +hwds diff --git a/src/satchip/generate_chips.py b/src/satchip/generate_chips.py index 40ada4e..23bd265 100644 --- a/src/satchip/generate_chips.py +++ b/src/satchip/generate_chips.py @@ -35,6 +35,7 @@ ALL_BANDS = ("B", "G", "R", "N", "SW1", "SW2", "Fmask") STACK_BANDS = ("B", "G", "R", "N", "SW1", "SW2") CHIP_BANDS = ("BANDS", "EVENT", "MASK", "Fmask") +SHOULD_CLEANUP = False def main(modalities: list[Modality]): @@ -45,97 +46,107 @@ def main(modalities: list[Modality]): "CHIPS_ALL": hwds_path / "CHIPS_ALL", "CHIPS": hwds_path / "CHIPS", "MERGED": hwds_path / "MERGED", + "PLOTS": hwds_path / 'PLOTS' } - for p in ("CHIPS", "CHIPS_ALL"): - shutil.rmtree(data_paths[p], ignore_errors=True) + if SHOULD_CLEANUP: + for item in data_paths['MERGED'].glob('*.tif'): + if item.is_dir(): + continue + + item.unlink() + + for p in ("CHIPS", "CHIPS_ALL"): + shutil.rmtree(data_paths[p], ignore_errors=True) for p in data_paths.values(): p.mkdir(parents=True, exist_ok=True) - gdf = _load_event_database_with_buffer(hwds_path) + gdf = _load_event_database(hwds_path) gdf_utm = gdf.to_crs(32615) gdf["buffered_event"] = gdf_utm.buffer(3000).to_crs(4326) gdf["buffered_event_background"] = gdf_utm.buffer(10000).to_crs(4326) + gdf = gdf.to_crs(4326) # keepers = [1442, 622, 1079, 628] - # keepers = [1442, 622] - # gdf = gdf[gdf["swathID"].isin(keepers)] - - earthaccess.login() - - tm_chips = [] - - for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): - swathID = f"{int(swath['swathID']):04d}" - print(f'Processing Swath {swathID} ({i} / {len(gdf)})') - - merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) - - template_path = merged[modalities[0].id]['BANDS'] - event_tif, mask_tif = _generate_masks( - template_path, swathID, swath - ) - - merged_data = { - **merged, - "EVENT": event_tif, - "MASK": mask_tif, - } - - breakpoint() - - if not all(is_valid_data(merged_data, modality) for modality in modalities): - print("Skipping: not enough valid data") - continue - - breakpoint() - - print("Chipping!") - chips = _chip_data(merged_data, data_paths) - - good_chips = filter_chips(chips) - print(f"Found {len(good_chips)} good chips") - - for chip in good_chips: - for band, chip_path in chip.items(): - if band not in ("MASK", "BANDS"): - continue - - dest = data_paths["CHIPS_TM"] / chip_path.name - shutil.copy(chip_path, dest) - - tm_chips += good_chips - + keepers = [1442, 622] + gdf = gdf[gdf["swathID"].isin(keepers)] + + # earthaccess.login() + + # tm_chips = [] + # + # for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): + # swathID = f"{int(swath['swathID']):04d}" + # print(f'Processing Swath {swathID} ({i} / {len(gdf)})') + # + # merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) + # + # template_path = merged[modalities[0].id]['BANDS'] + # event_tif, mask_tif = _generate_masks( + # template_path, swathID, swath + # ) + # + # merged_data = { + # **merged, + # "EVENT": event_tif, + # "MASK": mask_tif, + # } + # + # if not all(is_valid_data(merged_data, modality) for modality in modalities): + # print("Skipping: not enough valid data") + # continue + # + # for modality in modalities: + # print(f"Chipping {modality.id}!") + # chips = _chip_data(merged_data, data_paths['CHIPS_ALL'], modality) + # + # good_chips = filter_chips(chips, modality) + # print(f"Found {len(good_chips)} good chips") + # + # for chip in good_chips: + # for band, chip_path in chip.items(): + # if band not in ("MASK", "BANDS"): + # continue + # + # dest = data_paths["CHIPS"] / chip_path.name + # shutil.copy(chip_path, dest) + # + # tm_chips += good_chips + # for _, swath in gdf.iterrows(): swath_id = _make_swath_id(swath["swathID"]) - merged_file = list(data_paths["MERGED"].glob(f"{swath_id}.*BANDS.tif")) - if len(merged_file) == 0: - print(f"no chips for {swath_id}") - continue + for modality in modalities: + merged_file = list(data_paths["MERGED"].glob(f"{swath_id}.{modality.id}.*.BANDS.tif")) - all_chips = list(data_paths["CHIPS"].glob(f"*.{swath_id}.*.tif")) - good_chips = list(data_paths["CHIPS_TM"].glob(f"*.{swath_id}.*.tif")) + if len(merged_file) == 0: + print(f"no chips for {swath_id}") + continue - print(f"plotting {swath_id}") - _plot_chips( - merged_file[0], all_chips, good_chips, swath, save_to=data_paths["PLOTS"] - ) + all_chips = list(data_paths["CHIPS_ALL"].glob(f"*.{swath_id}.{modality.id}.*.tif")) + good_chips = list(data_paths["CHIPS"].glob(f"*.{swath_id}.{modality.id}.*.tif")) - band_chips = list(data_paths["CHIPS_TM"].glob("*BANDS.tif")) + print(f"plotting {swath_id}") + _plot_chips( + merged_file[0], all_chips, good_chips, swath, modality, save_to=data_paths["PLOTS"] + ) - means, stds = calculate_stats(chips=band_chips) + for modality in modalities: + print(f'Calulating stats for modality: {modality.id}') + band_chips = list(data_paths["CHIPS"].glob(f"*.{modality.id}.*.BANDS.tif")) - means_str = ", ".join(f"{x:.4f}" for x in means) - stds_str = ", ".join(f"{x:.4f}" for x in stds) + means, stds = calculate_stats(chips=band_chips, n_bands=len(modality.stack_bands)) - stats_str = ( - f"Means {STACK_BANDS}: {means_str}\n" - f"Stds {STACK_BANDS}: {stds_str}\n" - ) - (hwds_path / 'statistics.txt').write_text(stats_str) - print(stats_str) + means_str = ", ".join(f"{x:.4f}" for x in means) + stds_str = ", ".join(f"{x:.4f}" for x in stds) + + stats_str = ( + f"Means {modality.stack_bands}: {means_str}\n" + f"Stds {modality.stack_bands}: {stds_str}\n" + ) + (hwds_path / '{modality.id}-statistics.txt').write_text(stats_str) + print(stats_str) def _generate_masks( @@ -176,7 +187,7 @@ def _generate_masks( return event_path, mask_path -def _load_event_database_with_buffer(data_dir: Path): +def _load_event_database(data_dir: Path): # use 60-swath version hwds_google_drive_id = "1h_JIEcrrUF3OSTrmwAKNPa0eUEhPA2Xx" drive_url = f"https://drive.google.com/uc?id={hwds_google_drive_id}" @@ -255,6 +266,7 @@ def _plot_chips( all_chips, good_chips, swath, + modality, save_to: Path | None = None, quite=QUITE, ): @@ -265,7 +277,7 @@ def _plot_chips( full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] band_data = ds.read() - img = get_img(band_data) + img = get_img(band_data, modality) # plot BANDS and geom fig, ax = plt.subplots( @@ -326,11 +338,11 @@ def _make_swath_id(swathID): return f"{int(swathID):04d}" -def _chip_data(merged: dict[str, Path], data_paths: dict[str, Path], chip_size=CHIP_SIZE): +def _chip_data(merged, output_path: Path, modality: Modality, chip_size=CHIP_SIZE): chips = {} - grid = [] - with rasterio.open(merged["BANDS"]) as ref: + + with rasterio.open(merged[modality.id]["BANDS"]) as ref: n_cols = ref.width // chip_size n_rows = ref.height // chip_size @@ -343,8 +355,11 @@ def _chip_data(merged: dict[str, Path], data_paths: dict[str, Path], chip_size=C chips[tile_id] = {} grid.append((tile_id, bounds)) - for chip_layer in CHIP_BANDS: - layer_path = merged[chip_layer] + for chip_layer in modality.chip_bands: + if chip_layer in merged[modality.id]: + layer_path = merged[modality.id][chip_layer] + else: + layer_path = merged[chip_layer] with rasterio.open(layer_path) as src: for tile_id, bounds in grid: @@ -359,7 +374,7 @@ def _chip_data(merged: dict[str, Path], data_paths: dict[str, Path], chip_size=C data = src.read(window=window) if chip_layer == 'BANDS': - data = data_transform(data) + data = data_transform(data, modality) chip_meta = src.meta.copy() chip_meta.update( @@ -371,7 +386,7 @@ def _chip_data(merged: dict[str, Path], data_paths: dict[str, Path], chip_size=C ) chip_name = f"{tile_id}.{layer_path.name}" - chip_path = data_paths["CHIPS"] / chip_name + chip_path = output_path / chip_name with rasterio.open(chip_path, "w", **chip_meta) as dst: dst.write(data) @@ -386,34 +401,32 @@ def is_valid_data(merged, modality): if modality.id == 'HLS': is_valid = hls.is_valid_hls(merged_data['Fmask'], merged['EVENT']) - print('HLS', is_valid) elif modality.id == 'RTC': is_valid = opera_rtc.is_valid_rtc(merged_data['mask'], merged['EVENT']) - print('RTC', is_valid) return is_valid -def filter_chips(chips): - if MODALITY == 'HLS': +def filter_chips(chips, modality): + if modality.id == 'HLS': filtered_chips = hls.filter_hls_chips(chips) - elif MODALITY == 'RTC': + elif modality.id == 'RTC': filtered_chips = opera_rtc.filter_rtc_chips(chips) return filtered_chips -def get_img(band_data): - if MODALITY == 'HLS': +def get_img(band_data, modality): + if modality.id == 'HLS': img = hls.get_hls_img(band_data) - elif MODALITY == 'RTC': + elif modality.id == 'RTC': img = opera_rtc.get_rtc_img(band_data) return img -def data_transform(data): - if MODALITY == 'RTC': +def data_transform(data, modality): + if modality.id == 'RTC': data = 10 * np.log10(np.clip(data, 1e-10, None)) return data diff --git a/src/satchip/mosaic.py b/src/satchip/mosaic.py index b14e3fd..21e4eac 100644 --- a/src/satchip/mosaic.py +++ b/src/satchip/mosaic.py @@ -1,13 +1,15 @@ from pathlib import Path +import shutil import datetime import earthaccess import numpy as np import rasterio -from pyproj import Transformer from rasterio.crs import CRS from rasterio.merge import merge from rasterio.warp import calculate_default_transform, reproject +from rasterio.warp import Resampling, transform_bounds +from rasterio.transform import from_bounds from rasterio.mask import mask from shapely.geometry import box @@ -20,31 +22,24 @@ def data_over_swath(swath, modalities: list[Modality], output_path: Path): data_paths = { "RAW": output_path / "RAW", "REPROJECTED": output_path / "REPROJECTED", + "MOSAIC": output_path / "MOSAIC" } - for item in output_path.glob('*.tif'): - if item.is_dir(): - continue - - item.unlink() + for p in ("MOSAIC", ): + shutil.rmtree(data_paths[p], ignore_errors=True) for p in data_paths.values(): p.mkdir(parents=True, exist_ok=True) swathID = f"{int(swath['swathID']):04d}" - merged = {} - + stacked = {} for modality in modalities: print(f'Localizing data for {modality.id}.') bounding_box = swath["buffered_event_background"].bounds - # transformer = Transformer.from_crs(32615, 4326, always_xy=True) - # minx, miny = transformer.transform(minx, miny) - # maxx, maxy = transformer.transform(maxx, maxy) start_date = swath["ls5hlsDate"] if modality.id == 'HLS' else swath["s1Date"] - print(start_date) results = search_data(bounding_box, start_date, modality) @@ -66,20 +61,23 @@ def data_over_swath(swath, modalities: list[Modality], output_path: Path): merged_name = make_merge_name(swathID, start_date, band, modality) merged_band_path = _merge( - band_files, output_file=output_path / merged_name + band_files, output_file=data_paths['MOSAIC'] / merged_name ) - print(f'Clipping band {band} data to same area') - clipped_band_path = _clip_over_swath(merged_band_path, swath) - mod_merged[band] = clipped_band_path + mod_merged[band] = merged_band_path print(f'Stacking bands for {modality.id}') - stacked_data = _stack_bands(mod_merged, data_bands=modality.stack_bands) + stacked_data = _stack_bands(mod_merged, data_bands=modality.stack_bands, stacked_name='BANDS') + stacked[modality.id] = stacked_data - merged[modality.id] = stacked_data + print(f'Warp band {band} data to same area') + warped = _warp_over_swath( + data=stacked, + bounding_box_4326=bounding_box, + output_dir=output_path, + ) - breakpoint() - return merged + return warped def search_data(bounding_box: tuple, start_date: datetime.datetime, modality: Modality): @@ -184,13 +182,13 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: return output_file -def _stack_bands(merged: dict[str, Path], data_bands: tuple[str]) -> None: +def _stack_bands(merged: dict[str, Path], data_bands: tuple[str], stacked_name: str) -> None: with rasterio.open(merged[data_bands[0]]) as src: meta = src.meta.copy() band = data_bands[0] meta.update(count=len(data_bands), dtype=np.float32) - stacked_file_name = _rename(merged[band], f"{band}.tif", "BANDS.tif") + stacked_file_name = _rename(merged[band], f"{band}.tif", f"{stacked_name}.tif") with rasterio.open(stacked_file_name, "w", **meta) as dst: for idx, band in enumerate(data_bands, start=1): @@ -198,7 +196,7 @@ def _stack_bands(merged: dict[str, Path], data_bands: tuple[str]) -> None: dst.write(src.read(1), idx) - merged["BANDS"] = stacked_file_name + merged[stacked_name] = stacked_file_name return merged @@ -206,22 +204,84 @@ def _rename(path: Path, extension: str, mask_name: str) -> Path: return path.parent / path.name.replace(extension, mask_name) -def _clip_over_swath(input_path: Path, swath) -> Path: - bounding_box = swath["buffered_event_background"].bounds +def _warp_over_swath(data, bounding_box_4326, output_dir): + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + reference_path = next(iter(next(iter(data.values())).values())) + dst_transform, width, height, dst_crs = _build_common_grid( + bounding_box_4326, reference_path + ) + + output = {} + for sensor, bands in data.items(): + output[sensor] = {} + for band_name, input_path in bands.items(): + out_path = output_dir / input_path.name + _warp_single( + input_path, out_path, + dst_transform, width, height, dst_crs, + band_name + ) + output[sensor][band_name] = out_path + + return output + + +def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, band_name): + CATEGORICAL_BANDS = {"Fmask", "mask"} + resampling = Resampling.nearest if band_name in CATEGORICAL_BANDS else Resampling.bilinear with rasterio.open(input_path) as src: - out_image, out_transform = mask(src, shapes=[box(*bounding_box)], crop=True) - out_meta = src.meta.copy() + dst_data = np.zeros((src.count, height, width), dtype=src.dtypes[0]) + + reproject( + source=rasterio.band(src, list(range(1, src.count + 1))), + destination=dst_data, + src_transform=src.transform, + src_crs=src.crs, + dst_transform=dst_transform, + dst_crs=dst_crs, + resampling=resampling, + dst_nodata=src.nodata, + ) + out_meta = src.meta.copy() out_meta.update({ - "driver": "GTiff", - "height": out_image.shape[1], - "width": out_image.shape[2], - "transform": out_transform, - "crs": src.crs + "driver": "GTiff", + "height": height, + "width": width, + "transform": dst_transform, + "crs": dst_crs, }) - with rasterio.open(input_path, "w", **out_meta) as dest: - dest.write(out_image) + with rasterio.open(output_path, "w", **out_meta) as dest: + dest.write(dst_data) + + return output_path + + +def _build_common_grid(bounding_box_4326, reference_path): + dst_crs = CRS.from_epsg(4326) + minx, miny, maxx, maxy = bounding_box_4326 + + with rasterio.open(reference_path) as ref: + bounds_4326 = transform_bounds(ref.crs, dst_crs, *ref.bounds, densify_pts=21) + ref_width = ref.width + ref_height = ref.height + + ref_bbox_width = bounds_4326[2] - bounds_4326[0] + ref_bbox_height = bounds_4326[3] - bounds_4326[1] + res_x = ref_bbox_width / ref_width + res_y = ref_bbox_height / ref_height + + minx = np.floor(minx / res_x) * res_x + miny = np.floor(miny / res_y) * res_y + maxx = np.ceil(maxx / res_x) * res_x + maxy = np.ceil(maxy / res_y) * res_y + + width = int(round((maxx - minx) / res_x)) + height = int(round((maxy - miny) / res_y)) + dst_transform = from_bounds(minx, miny, maxx, maxy, width, height) - return input_path + return dst_transform, width, height, dst_crs From 2f087fec973a5fd84e2eecb784f2893341764fce Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 17 Mar 2026 15:48:20 -0800 Subject: [PATCH 03/21] Uncomment chip generation --- src/satchip/generate_chips.py | 84 +++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/satchip/generate_chips.py b/src/satchip/generate_chips.py index 23bd265..b53a00f 100644 --- a/src/satchip/generate_chips.py +++ b/src/satchip/generate_chips.py @@ -72,48 +72,48 @@ def main(modalities: list[Modality]): keepers = [1442, 622] gdf = gdf[gdf["swathID"].isin(keepers)] - # earthaccess.login() - - # tm_chips = [] - # - # for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): - # swathID = f"{int(swath['swathID']):04d}" - # print(f'Processing Swath {swathID} ({i} / {len(gdf)})') - # - # merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) - # - # template_path = merged[modalities[0].id]['BANDS'] - # event_tif, mask_tif = _generate_masks( - # template_path, swathID, swath - # ) - # - # merged_data = { - # **merged, - # "EVENT": event_tif, - # "MASK": mask_tif, - # } - # - # if not all(is_valid_data(merged_data, modality) for modality in modalities): - # print("Skipping: not enough valid data") - # continue - # - # for modality in modalities: - # print(f"Chipping {modality.id}!") - # chips = _chip_data(merged_data, data_paths['CHIPS_ALL'], modality) - # - # good_chips = filter_chips(chips, modality) - # print(f"Found {len(good_chips)} good chips") - # - # for chip in good_chips: - # for band, chip_path in chip.items(): - # if band not in ("MASK", "BANDS"): - # continue - # - # dest = data_paths["CHIPS"] / chip_path.name - # shutil.copy(chip_path, dest) - # - # tm_chips += good_chips - # + earthaccess.login() + + tm_chips = [] + + for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): + swathID = f"{int(swath['swathID']):04d}" + print(f'Processing Swath {swathID} ({i} / {len(gdf)})') + + merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) + + template_path = merged[modalities[0].id]['BANDS'] + event_tif, mask_tif = _generate_masks( + template_path, swathID, swath + ) + + merged_data = { + **merged, + "EVENT": event_tif, + "MASK": mask_tif, + } + + if not all(is_valid_data(merged_data, modality) for modality in modalities): + print("Skipping: not enough valid data") + continue + + for modality in modalities: + print(f"Chipping {modality.id}!") + chips = _chip_data(merged_data, data_paths['CHIPS_ALL'], modality) + + good_chips = filter_chips(chips, modality) + print(f"Found {len(good_chips)} good chips") + + for chip in good_chips: + for band, chip_path in chip.items(): + if band not in ("MASK", "BANDS"): + continue + + dest = data_paths["CHIPS"] / chip_path.name + shutil.copy(chip_path, dest) + + tm_chips += good_chips + for _, swath in gdf.iterrows(): swath_id = _make_swath_id(swath["swathID"]) From 9e905c3da6beab77ab39b4e60d340fd634af6b2c Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 27 Mar 2026 14:10:02 -0800 Subject: [PATCH 04/21] Reworking models and data download --- src/satchip/download_data.py | 38 ++++++++++++++++++ src/satchip/merge_modality.py | 2 + src/satchip/models.py | 75 +++++++++++++++++++++++++++++++++++ tests/conftest.py | 63 +++++++++++++++++++++++++++++ tests/test_download_data.py | 18 +++++++++ tests/test_merge_modality.py | 5 +++ 6 files changed, 201 insertions(+) create mode 100644 src/satchip/download_data.py create mode 100644 src/satchip/merge_modality.py create mode 100644 src/satchip/models.py create mode 100644 tests/conftest.py create mode 100644 tests/test_download_data.py create mode 100644 tests/test_merge_modality.py diff --git a/src/satchip/download_data.py b/src/satchip/download_data.py new file mode 100644 index 0000000..c2723f2 --- /dev/null +++ b/src/satchip/download_data.py @@ -0,0 +1,38 @@ +from datetime import timedelta +from pathlib import Path + +import earthaccess + +from satchip import models + + +def download_data(event: models.Event, modality: models.Modality, download_path: Path) -> list[Path]: + if not earthaccess.__auth__.authenticated: + print('Logging in to earthaccess') + earthaccess.login() + + results = _search_data(event, modality) + + if not results: + return [] + + local_files = earthaccess.download( + results, local_path=download_path, show_progress=True + ) + + return local_files + + +def _search_data(event: models.Event, modality: models.Modality) -> list[earthaccess.DataGranule]: + start_date = event.date + final_date = start_date + timedelta(days=1) + + collection_id = modality.collection + + results = earthaccess.search_data( + short_name=[collection_id], + temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + bounding_box=event.wgs84_geometry.bounds, + ) + + return results diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py new file mode 100644 index 0000000..3b32c40 --- /dev/null +++ b/src/satchip/merge_modality.py @@ -0,0 +1,2 @@ +def merge_modality(): + pass diff --git a/src/satchip/models.py b/src/satchip/models.py new file mode 100644 index 0000000..b592610 --- /dev/null +++ b/src/satchip/models.py @@ -0,0 +1,75 @@ +from datetime import datetime +from dataclasses import dataclass +from typing import NamedTuple +from typing import Tuple + +import shapely + + +class Band(NamedTuple): + id: str + name: str + short: str + + +@dataclass(frozen=True) +class Modality: + id: str + bands: Tuple[Band, ...] + + collection: str + + +@dataclass(frozen=True) +class Event: + name: str + date: datetime + wgs84_geometry: shapely.polygon + + +RTC_BANDS = ( + Band("VV", "VV", "VV"), + Band("VH", "VH", "VH"), + Band("mask", "Validitiy Mask", "mask"), +) + +# https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/ +OPERA_RTC = Modality( + id="RTC", + bands=RTC_BANDS, + collection="OPERA_L2_RTC-S1_V1" +) + + +HLS_S30_BANDS = ( + Band("B02", "Blue", "B"), + Band("B03", "Green", "G"), + Band("B04", "Red", "R"), + Band("B8A", "NIR Narrow", "N"), + Band("B11", "SWIR 1", "SW1"), + Band("B12", "SWIR 2", "SW2"), + Band("Fmask", "Cloud Mask", "Fmask"), +) + +# https://www.earthdata.nasa.gov/data/projects/hls/spectral-bands +HLS_S30 = Modality( + id="HLS_S30", + bands=HLS_S30_BANDS, + collection="HLSS30" +) + +HLS_L30_BANDS = ( + Band("B02", "Blue", "B"), + Band("B03", "Green", "G"), + Band("B04", "Red", "R"), + Band("B05", "NIR Narrow", "N"), + Band("B06", "SWIR 1", "SW1"), + Band("B07", "SWIR 2", "SW2"), + Band("Fmask", "Cloud Mask", "Fmask"), +) + +HLS_L30 = Modality( + id="HLS_L30", + bands=HLS_L30_BANDS, + collection="HLSL30" +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e7c168d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,63 @@ +from datetime import datetime +from pathlib import Path + +import pytest +import geopandas as gpd +import pandas as pd +from shapely import wkt + +from satchip import download_data, models + +DATA_PATH = Path(__file__).parent / "data" + + +def pytest_addoption(parser): + parser.addoption("--download", action="store_true", default=False) + + +def pytest_configure(config): + config.addinivalue_line("markers", "download: marks tests that download data") + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--download"): + skip = pytest.mark.skip(reason="pass --download to run") + for item in items: + if item.get_closest_marker("download"): + item.add_marker(skip) + + +@pytest.fixture +def pristine_gdf(tmp_path): + shp_path = DATA_PATH / "hwds_pristine" + df = gpd.read_file(shp_path) + + df["SwathDate"] = pd.to_datetime(df["SwathDate"], format="%Y-%m-%d") + df["HLSDate"] = pd.to_datetime(df["HLSDate"], format="%Y-%m-%d") + + return df + + +@pytest.fixture(scope="session") +def s2_local_files(s2_event): + download_path = DATA_PATH / "raw" + print('Downloading test data...') + + local_files = download_data.download_data(s2_event, models.HLS_S30, download_path) + + return local_files + + +@pytest.fixture(scope="session") +def s2_event() -> models.Event: + geom = wkt.loads('POLYGON Z ((-97.46296108799999 41.72598898100006 0, -97.45654663199997 41.72832979400005 0, -97.45286506599996 41.735911517000034 0, -97.44692854399995 41.73806293200005 0, -97.44255668499994 41.741031193000026 0, -97.43649360699999 41.746392472000025 0, -97.43643518499994 41.74966361400004 0, -97.44477798299994 41.751269181000055 0, -97.44546922799998 41.757304729000055 0, -97.44413504799996 41.76092731800003 0, -97.43601483299994 41.761099638000076 0, -97.43493753799999 41.76400779800008 0, -97.43574582899998 41.76925407400006 0, -97.43507482199999 41.775018801000044 0, -97.42929198599995 41.77529913700005 0, -97.42434097299997 41.77189406900004 0, -97.41643891099994 41.77095250000008 0, -97.40855823399994 41.772049811000045 0, -97.40518880099995 41.769938841000055 0, -97.40212726299995 41.772478093000075 0, -97.39708935799996 41.77557899800007 0, -97.39216022699998 41.77683695500008 0, -97.38494019299998 41.779687625000065 0, -97.37802386699997 41.78005042400008 0, -97.37595922599996 41.772699628000055 0, -97.36870573799996 41.772803176000025 0, -97.36473372899997 41.776025134000065 0, -97.35972290999996 41.77556353600005 0, -97.35569283499996 41.77524485300006 0, -97.35141833499995 41.77176253300007 0, -97.35179576499996 41.769616866000035 0, -97.35247585899998 41.76537635100004 0, -97.35455601099994 41.765104164000036 0, -97.35748216699994 41.763087014000064 0, -97.35755091099998 41.760709776000056 0, -97.35659214299994 41.75815047000003 0, -97.34941309099997 41.75410074900003 0, -97.34425890099999 41.75299627900006 0, -97.33955929299998 41.75280198300004 0, -97.33427827799994 41.75626863400004 0, -97.32638406299998 41.75688761500004 0, -97.32401029299996 41.75030413500008 0, -97.32617321299995 41.74390741500008 0, -97.33893306699997 41.74281858000006 0, -97.33915551999996 41.73881615700003 0, -97.33542314499994 41.73642923700004 0, -97.33487090999995 41.73053873300006 0, -97.33494400699999 41.72165371600005 0, -97.34530133099997 41.72321647500007 0, -97.34362613199994 41.72861741600008 0, -97.34850260899998 41.72835093900005 0, -97.34904493499994 41.72262336800003 0, -97.34959717099997 41.71581247300003 0, -97.35659214299994 41.713051300000075 0, -97.35972147399997 41.70716079600004 0, -97.36818907299994 41.707897109000044 0, -97.37554466699999 41.70552724600003 0, -97.37794521999996 41.71268314300005 0, -97.38066826999994 41.713925654000036 0, -97.38197931699995 41.71100923000006 0, -97.38053398199997 41.702322166000044 0, -97.37438188299996 41.70104730000003 0, -97.37256136399998 41.69851681000006 0, -97.37141382499999 41.691215095000075 0, -97.37359654199997 41.685445995000066 0, -97.37716937399995 41.686764063000055 0, -97.37836702299995 41.692105638000044 0, -97.38678588699997 41.69234998300004 0, -97.38711030299999 41.69474924900004 0, -97.38893657299997 41.696020211000075 0, -97.39203310699997 41.69605112600004 0, -97.39708425799995 41.69660031600006 0, -97.39699799999994 41.69790345900003 0, -97.39728821299997 41.70098871600004 0, -97.39958472399996 41.70204029100006 0, -97.40051777999997 41.703388508000046 0, -97.40357986399994 41.71025483200003 0, -97.40555695699999 41.71360353400007 0, -97.41463431499994 41.71473101400005 0, -97.41599053299996 41.71041910900004 0, -97.40847919899994 41.70361729000007 0, -97.40905360899995 41.698218834000045 0, -97.42033287699996 41.69235264400004 0, -97.42815022299999 41.69435335000003 0, -97.42415654299998 41.69969951000007 0, -97.42249215599998 41.70624040400003 0, -97.42727819099997 41.70936973400006 0, -97.43331385799996 41.70941182400003 0, -97.43942735499996 41.70716079600004 0, -97.44623824999996 41.70182252600006 0, -97.45065612799999 41.700902135000035 0, -97.45614052299999 41.70244454500005 0, -97.45855998799999 41.70632093900008 0, -97.46176883199996 41.71106968300006 0, -97.46907545699997 41.71252207500004 0, -97.47778018699995 41.71142744600007 0, -97.48360613399996 41.71139459500006 0, -97.48556606699998 41.71563875600003 0, -97.47960867399996 41.71848116100006 0, -97.47027376699998 41.720714549000036 0, -97.46593462299995 41.726661585000045 0, -97.46296108799999 41.72598898100006 0))') + name = '95a', + date = datetime(year=2018, month=7, day=2) + + event = models.Event( + name=name, + date=date, + wgs84_geometry=geom, + ) + + return event diff --git a/tests/test_download_data.py b/tests/test_download_data.py new file mode 100644 index 0000000..7ded134 --- /dev/null +++ b/tests/test_download_data.py @@ -0,0 +1,18 @@ +import pytest + +from satchip import models +from satchip import download_data + + +def test_models(): + assert len(models.HLS_S30_BANDS) == 7 + assert len(models.HLS_L30_BANDS) == 7 + + +@pytest.mark.download +def test_download_data(s2_event, tmp_path): + local_files = download_data.download_data(s2_event, models.HLS_S30, tmp_path) + assert len(local_files) > 1 + + local_files = download_data.download_data(s2_event, models.HLS_L30, tmp_path) + assert len(local_files) == 0 diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py new file mode 100644 index 0000000..70a2751 --- /dev/null +++ b/tests/test_merge_modality.py @@ -0,0 +1,5 @@ +from satchip import merge_modality + + +def test_merge_modality(s2_local_files): + print(merge_modality.merge_modality) From a61d7ef7444156b21d51abf7a690c32cca61bafc Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 27 Mar 2026 15:44:57 -0800 Subject: [PATCH 05/21] Start work on merge data function --- src/satchip/merge_modality.py | 11 +++++++++-- src/satchip/models.py | 24 +++++++++++++++++------- tests/test_download_data.py | 5 ----- tests/test_merge_modality.py | 11 ++++++++--- tests/test_models.py | 12 ++++++++++++ 5 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 tests/test_models.py diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 3b32c40..fc727d0 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -1,2 +1,9 @@ -def merge_modality(): - pass +from pathlib import Path + +from satchip import models + + +def merge_modality(band_tifs: list[Path], event: models.Event, band: models.Band, modality: models.Modality): + if len(band_tifs) == 0: + print(f"Warning: no data for {event.name}") + return [] diff --git a/src/satchip/models.py b/src/satchip/models.py index b592610..9f05627 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -9,7 +9,11 @@ class Band(NamedTuple): id: str name: str - short: str + shortname: str + + +def bands_by_shortname(bands: tuple[Band, ...]) -> dict[str, Band]: + return {band.shortname: band for band in bands} @dataclass(frozen=True) @@ -27,21 +31,23 @@ class Event: wgs84_geometry: shapely.polygon -RTC_BANDS = ( +OPERA_RTC_BANDS_TUPLE = ( Band("VV", "VV", "VV"), Band("VH", "VH", "VH"), Band("mask", "Validitiy Mask", "mask"), ) +OPERA_RTC_BANDS = bands_by_shortname(OPERA_RTC_BANDS_TUPLE) + # https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/ OPERA_RTC = Modality( id="RTC", - bands=RTC_BANDS, + bands=OPERA_RTC_BANDS_TUPLE, collection="OPERA_L2_RTC-S1_V1" ) -HLS_S30_BANDS = ( +HLS_S30_BANDS_TUPLE = ( Band("B02", "Blue", "B"), Band("B03", "Green", "G"), Band("B04", "Red", "R"), @@ -51,14 +57,16 @@ class Event: Band("Fmask", "Cloud Mask", "Fmask"), ) +HLS_S30_BANDS = bands_by_shortname(HLS_S30_BANDS_TUPLE) + # https://www.earthdata.nasa.gov/data/projects/hls/spectral-bands HLS_S30 = Modality( id="HLS_S30", - bands=HLS_S30_BANDS, + bands=HLS_S30_BANDS_TUPLE, collection="HLSS30" ) -HLS_L30_BANDS = ( +HLS_L30_BANDS_TUPLE = ( Band("B02", "Blue", "B"), Band("B03", "Green", "G"), Band("B04", "Red", "R"), @@ -68,8 +76,10 @@ class Event: Band("Fmask", "Cloud Mask", "Fmask"), ) +HLS_L30_BANDS = bands_by_shortname(HLS_L30_BANDS_TUPLE) + HLS_L30 = Modality( id="HLS_L30", - bands=HLS_L30_BANDS, + bands=HLS_L30_BANDS_TUPLE, collection="HLSL30" ) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 7ded134..e46c856 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -4,11 +4,6 @@ from satchip import download_data -def test_models(): - assert len(models.HLS_S30_BANDS) == 7 - assert len(models.HLS_L30_BANDS) == 7 - - @pytest.mark.download def test_download_data(s2_event, tmp_path): local_files = download_data.download_data(s2_event, models.HLS_S30, tmp_path) diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 70a2751..d7b8c8c 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,5 +1,10 @@ -from satchip import merge_modality +from satchip import merge_modality, models -def test_merge_modality(s2_local_files): - print(merge_modality.merge_modality) +def test_merge_s2_modality(s2_local_files, s2_event): + empty_result = merge_modality.merge_modality([], s2_event, models.HLS_S30_BANDS['R'], models.HLS_S30) + assert len(empty_result) == 0 + + result = merge_modality.merge_modality(s2_local_files, s2_event, models.HLS_S30_BANDS['R'], models.HLS_S30) + + breakpoint() diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..ce60ee1 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,12 @@ +from satchip import models + + +def test_hls_model_len(): + assert len(models.HLS_S30_BANDS_TUPLE) == 7 + assert len(models.HLS_L30_BANDS_TUPLE) == 7 + + +def test_hls_model_names(): + assert models.HLS_S30_BANDS['N'].id != models.HLS_L30_BANDS['N'].id + assert models.HLS_S30_BANDS['SW1'].id != models.HLS_L30_BANDS['SW1'].id + assert models.HLS_S30_BANDS['SW2'].id != models.HLS_L30_BANDS['SW2'].id From 1055f2408b4a6bf9fc078a97e16570f4a7ce7438 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 31 Mar 2026 13:47:01 -0800 Subject: [PATCH 06/21] Working on adding merging code --- src/satchip/download_data.py | 2 +- src/satchip/merge_modality.py | 66 +++++++++++++++- src/satchip/models.py | 139 +++++++++++++++++++++------------- tests/conftest.py | 2 +- tests/test_merge_modality.py | 19 ++++- tests/test_models.py | 41 +++++++++- 6 files changed, 208 insertions(+), 61 deletions(-) diff --git a/src/satchip/download_data.py b/src/satchip/download_data.py index c2723f2..cb646c4 100644 --- a/src/satchip/download_data.py +++ b/src/satchip/download_data.py @@ -27,7 +27,7 @@ def _search_data(event: models.Event, modality: models.Modality) -> list[earthac start_date = event.date final_date = start_date + timedelta(days=1) - collection_id = modality.collection + collection_id = modality['collection'] results = earthaccess.search_data( short_name=[collection_id], diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index fc727d0..74fb26e 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -1,9 +1,71 @@ from pathlib import Path +import datetime + +import numpy as np +import rasterio +from rasterio.merge import merge from satchip import models -def merge_modality(band_tifs: list[Path], event: models.Event, band: models.Band, modality: models.Modality): - if len(band_tifs) == 0: +def merge_modality(modality_files: list[Path], modality: models.Modality, event: models.Event, output_path: Path, selected_bands: list[models.Band] | None = None): + if len(modality_files) == 0: print(f"Warning: no data for {event.name}") return [] + + merged = {} + + if selected_bands is None: + selected_bands = modality['bands'] + + for band in selected_bands: + band_files = [f for f in modality_files if band.id in models.band_id_from_filename(f.name, modality['id'])] + + merged_name = _make_merge_name(event.name, event.date, band.shortname, modality['id']) + + merged_band_path = _merge( + band_files, output_file=output_path / merged_name + ) + + merged[band] = merged_band_path + + return merged + + +def _make_merge_name(event_name: str, start_date: datetime.datetime, band: str, modality_id: str): + date_str = start_date.date().isoformat() + + return f'{event_name}.{modality_id}.{date_str}.{band}.tif' + + +def _merge(band_files: list[Path], output_file: Path) -> Path: + band_datasets = [rasterio.open(band_file) for band_file in band_files] + + reference_crs = band_datasets[0].crs + for ds in band_datasets[1:]: + if ds.crs != reference_crs: + ds.crs = reference_crs + + try: + mosaic, out_trans = merge(band_datasets) + mosaic = np.squeeze(mosaic) + + out_meta = band_datasets[0].meta.copy() + + out_meta.update( + { + "driver": "GTiff", + "height": mosaic.shape[0], + "width": mosaic.shape[1], + "transform": out_trans, + "crs": band_datasets[0].crs, + } + ) + + with rasterio.open(output_file, "w", **out_meta) as dst: + dst.write(mosaic, 1) + finally: + for ds in band_datasets: + ds.close() + + return output_file diff --git a/src/satchip/models.py b/src/satchip/models.py index 9f05627..f176b87 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -1,6 +1,6 @@ from datetime import datetime from dataclasses import dataclass -from typing import NamedTuple +from typing import TypedDict, NamedTuple from typing import Tuple import shapely @@ -16,11 +16,13 @@ def bands_by_shortname(bands: tuple[Band, ...]) -> dict[str, Band]: return {band.shortname: band for band in bands} -@dataclass(frozen=True) -class Modality: +def bands_by_id(bands: tuple[Band, ...]) -> dict[str, Band]: + return {band.id: band for band in bands} + + +class Modality(TypedDict): id: str bands: Tuple[Band, ...] - collection: str @@ -28,58 +30,93 @@ class Modality: class Event: name: str date: datetime - wgs84_geometry: shapely.polygon + wgs84_geometry: shapely.geometry.Polygon + + +class ModalityError(Exception): + pass + + +MODALITIES: dict[str, Modality] = { + "OPERA_RTC": { + "id": "OPERA_RTC", + "collection": "OPERA_L2_RTC-S1_V1", + "bands": ( + Band("VV", "VV", "VV"), + Band("VH", "VH", "VH"), + Band("mask", "Validitiy Mask", "mask"), + ) + }, + "HLS_S30": { + "id": "HLS_S30", + "collection": "HLSS30", + "bands": ( + Band("B02", "Blue", "B"), + Band("B03", "Green", "G"), + Band("B04", "Red", "R"), + Band("B8A", "NIR Narrow", "N"), + Band("B11", "SWIR 1", "SW1"), + Band("B12", "SWIR 2", "SW2"), + Band("Fmask", "Cloud Mask", "Fmask"), + ) + }, + "HLS_L30": { + "id": "HLS_L30", + "collection": "HLSL30", + "bands": ( + Band("B02", "Blue", "B"), + Band("B03", "Green", "G"), + Band("B04", "Red", "R"), + Band("B05", "NIR Narrow", "N"), + Band("B06", "SWIR 1", "SW1"), + Band("B07", "SWIR 2", "SW2"), + Band("Fmask", "Cloud Mask", "Fmask"), + ) + } +} + +MODALITY_IDS = list(MODALITIES.keys()) +# https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/ +OPERA_RTC = MODALITIES['OPERA_RTC'] +OPERA_RTC_BANDS = bands_by_shortname(MODALITIES['OPERA_RTC']['bands']) -OPERA_RTC_BANDS_TUPLE = ( - Band("VV", "VV", "VV"), - Band("VH", "VH", "VH"), - Band("mask", "Validitiy Mask", "mask"), -) +# https://www.earthdata.nasa.gov/data/projects/hls/spectral-bands +HLS_S30 = MODALITIES['HLS_S30'] +HLS_S30_BANDS = bands_by_shortname(MODALITIES['HLS_S30']['bands']) -OPERA_RTC_BANDS = bands_by_shortname(OPERA_RTC_BANDS_TUPLE) +HLS_L30 = MODALITIES['HLS_L30'] +HLS_L30_BANDS = bands_by_shortname(MODALITIES['HLS_L30']['bands']) -# https://hyp3-docs.asf.alaska.edu/guides/opera_rtc_product_guide/ -OPERA_RTC = Modality( - id="RTC", - bands=OPERA_RTC_BANDS_TUPLE, - collection="OPERA_L2_RTC-S1_V1" -) +def band_id_from_filename(filename: str, modality_id: str) -> Band | None: + if 'HLS' in modality_id: + sensor, band = band_from_hls_filename(filename) + elif 'RTC' in modality_id: + sensor, band = band_from_rtc_filename(filename) + else: + raise ModalityError(f'Modality not found {modality_id}, must be ({MODALITY_IDS})') -HLS_S30_BANDS_TUPLE = ( - Band("B02", "Blue", "B"), - Band("B03", "Green", "G"), - Band("B04", "Red", "R"), - Band("B8A", "NIR Narrow", "N"), - Band("B11", "SWIR 1", "SW1"), - Band("B12", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "Fmask"), -) + if sensor not in modality_id: + return '' -HLS_S30_BANDS = bands_by_shortname(HLS_S30_BANDS_TUPLE) + bands = bands_by_id(MODALITIES[modality_id]['bands']) -# https://www.earthdata.nasa.gov/data/projects/hls/spectral-bands -HLS_S30 = Modality( - id="HLS_S30", - bands=HLS_S30_BANDS_TUPLE, - collection="HLSS30" -) - -HLS_L30_BANDS_TUPLE = ( - Band("B02", "Blue", "B"), - Band("B03", "Green", "G"), - Band("B04", "Red", "R"), - Band("B05", "NIR Narrow", "N"), - Band("B06", "SWIR 1", "SW1"), - Band("B07", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "Fmask"), -) - -HLS_L30_BANDS = bands_by_shortname(HLS_L30_BANDS_TUPLE) - -HLS_L30 = Modality( - id="HLS_L30", - bands=HLS_L30_BANDS_TUPLE, - collection="HLSL30" -) + try: + return bands[band] + except KeyError: + return '' + + +def band_from_hls_filename(filename: str) -> tuple[str, str]: + # HLS.L30.T15TUG.2017167T165321.v2.0.B06.tif + parts = filename.split('.') + + sensor_key, band = parts[1], parts[-2] + + return sensor_key, band + + +def band_from_rtc_filename(filename: str) -> tuple[str, str]: + # OPERA_L2_RTC-S1_T063-133415-IW2_20170620T001327Z_20250925T045340Z_S1A_30_v1.0_VV.tif + return 'RTC', filename.split('_')[-1].split('.')[0] diff --git a/tests/conftest.py b/tests/conftest.py index e7c168d..e521051 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,7 +51,7 @@ def s2_local_files(s2_event): @pytest.fixture(scope="session") def s2_event() -> models.Event: geom = wkt.loads('POLYGON Z ((-97.46296108799999 41.72598898100006 0, -97.45654663199997 41.72832979400005 0, -97.45286506599996 41.735911517000034 0, -97.44692854399995 41.73806293200005 0, -97.44255668499994 41.741031193000026 0, -97.43649360699999 41.746392472000025 0, -97.43643518499994 41.74966361400004 0, -97.44477798299994 41.751269181000055 0, -97.44546922799998 41.757304729000055 0, -97.44413504799996 41.76092731800003 0, -97.43601483299994 41.761099638000076 0, -97.43493753799999 41.76400779800008 0, -97.43574582899998 41.76925407400006 0, -97.43507482199999 41.775018801000044 0, -97.42929198599995 41.77529913700005 0, -97.42434097299997 41.77189406900004 0, -97.41643891099994 41.77095250000008 0, -97.40855823399994 41.772049811000045 0, -97.40518880099995 41.769938841000055 0, -97.40212726299995 41.772478093000075 0, -97.39708935799996 41.77557899800007 0, -97.39216022699998 41.77683695500008 0, -97.38494019299998 41.779687625000065 0, -97.37802386699997 41.78005042400008 0, -97.37595922599996 41.772699628000055 0, -97.36870573799996 41.772803176000025 0, -97.36473372899997 41.776025134000065 0, -97.35972290999996 41.77556353600005 0, -97.35569283499996 41.77524485300006 0, -97.35141833499995 41.77176253300007 0, -97.35179576499996 41.769616866000035 0, -97.35247585899998 41.76537635100004 0, -97.35455601099994 41.765104164000036 0, -97.35748216699994 41.763087014000064 0, -97.35755091099998 41.760709776000056 0, -97.35659214299994 41.75815047000003 0, -97.34941309099997 41.75410074900003 0, -97.34425890099999 41.75299627900006 0, -97.33955929299998 41.75280198300004 0, -97.33427827799994 41.75626863400004 0, -97.32638406299998 41.75688761500004 0, -97.32401029299996 41.75030413500008 0, -97.32617321299995 41.74390741500008 0, -97.33893306699997 41.74281858000006 0, -97.33915551999996 41.73881615700003 0, -97.33542314499994 41.73642923700004 0, -97.33487090999995 41.73053873300006 0, -97.33494400699999 41.72165371600005 0, -97.34530133099997 41.72321647500007 0, -97.34362613199994 41.72861741600008 0, -97.34850260899998 41.72835093900005 0, -97.34904493499994 41.72262336800003 0, -97.34959717099997 41.71581247300003 0, -97.35659214299994 41.713051300000075 0, -97.35972147399997 41.70716079600004 0, -97.36818907299994 41.707897109000044 0, -97.37554466699999 41.70552724600003 0, -97.37794521999996 41.71268314300005 0, -97.38066826999994 41.713925654000036 0, -97.38197931699995 41.71100923000006 0, -97.38053398199997 41.702322166000044 0, -97.37438188299996 41.70104730000003 0, -97.37256136399998 41.69851681000006 0, -97.37141382499999 41.691215095000075 0, -97.37359654199997 41.685445995000066 0, -97.37716937399995 41.686764063000055 0, -97.37836702299995 41.692105638000044 0, -97.38678588699997 41.69234998300004 0, -97.38711030299999 41.69474924900004 0, -97.38893657299997 41.696020211000075 0, -97.39203310699997 41.69605112600004 0, -97.39708425799995 41.69660031600006 0, -97.39699799999994 41.69790345900003 0, -97.39728821299997 41.70098871600004 0, -97.39958472399996 41.70204029100006 0, -97.40051777999997 41.703388508000046 0, -97.40357986399994 41.71025483200003 0, -97.40555695699999 41.71360353400007 0, -97.41463431499994 41.71473101400005 0, -97.41599053299996 41.71041910900004 0, -97.40847919899994 41.70361729000007 0, -97.40905360899995 41.698218834000045 0, -97.42033287699996 41.69235264400004 0, -97.42815022299999 41.69435335000003 0, -97.42415654299998 41.69969951000007 0, -97.42249215599998 41.70624040400003 0, -97.42727819099997 41.70936973400006 0, -97.43331385799996 41.70941182400003 0, -97.43942735499996 41.70716079600004 0, -97.44623824999996 41.70182252600006 0, -97.45065612799999 41.700902135000035 0, -97.45614052299999 41.70244454500005 0, -97.45855998799999 41.70632093900008 0, -97.46176883199996 41.71106968300006 0, -97.46907545699997 41.71252207500004 0, -97.47778018699995 41.71142744600007 0, -97.48360613399996 41.71139459500006 0, -97.48556606699998 41.71563875600003 0, -97.47960867399996 41.71848116100006 0, -97.47027376699998 41.720714549000036 0, -97.46593462299995 41.726661585000045 0, -97.46296108799999 41.72598898100006 0))') - name = '95a', + name = '95a' date = datetime(year=2018, month=7, day=2) event = models.Event( diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index d7b8c8c..3572bae 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,10 +1,21 @@ from satchip import merge_modality, models -def test_merge_s2_modality(s2_local_files, s2_event): - empty_result = merge_modality.merge_modality([], s2_event, models.HLS_S30_BANDS['R'], models.HLS_S30) +def test_make_merge_name(s2_event): + merged_name = merge_modality._make_merge_name(s2_event.name, s2_event.date, 'BAND', 'MOD') + + assert 'BAND' in merged_name + assert 'MOD' in merged_name + assert s2_event.name in merged_name + assert merged_name.endswith('.tif') + + +def test_merge_s2_modality_empty(s2_local_files, s2_event, tmp_path): + empty_result = merge_modality.merge_modality([], models.HLS_S30, s2_event, tmp_path) assert len(empty_result) == 0 - result = merge_modality.merge_modality(s2_local_files, s2_event, models.HLS_S30_BANDS['R'], models.HLS_S30) - breakpoint() +def test_merge_s2_modality(s2_local_files, s2_event, tmp_path): + result = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path) + + assert result diff --git a/tests/test_models.py b/tests/test_models.py index ce60ee1..116067e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,12 +1,49 @@ +import pytest + from satchip import models def test_hls_model_len(): - assert len(models.HLS_S30_BANDS_TUPLE) == 7 - assert len(models.HLS_L30_BANDS_TUPLE) == 7 + assert len(models.HLS_S30_BANDS) == 7 + assert len(models.HLS_L30_BANDS) == 7 def test_hls_model_names(): assert models.HLS_S30_BANDS['N'].id != models.HLS_L30_BANDS['N'].id assert models.HLS_S30_BANDS['SW1'].id != models.HLS_L30_BANDS['SW1'].id assert models.HLS_S30_BANDS['SW2'].id != models.HLS_L30_BANDS['SW2'].id + + +@pytest.mark.parametrize("filename, expected_band", [ + ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_VH.tif', 'VH'), + ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_VH.tif', 'VH'), + ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_VV.tif', 'VV'), + ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_VV.tif', 'VV'), + ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_mask.tif', 'mask'), + ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_mask.tif', 'mask'), +]) +def test_band_id_from_filename_rtc(filename, expected_band): + assert models.band_id_from_filename(filename, 'OPERA_RTC').id == expected_band + + +@pytest.mark.parametrize("filename, expected_band", [ + ('HLS.S30.T13TFL.2019187T174919.v2.0.B12.tif', 'B12'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.B11.tif', 'B11'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.B8A.tif', 'B8A'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.B04.tif', 'B04'), + ('HLS.S30.T13TFL.2019187T174919.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.B12.tif', 'B12'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.B11.tif', 'B11'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.B8A.tif', 'B8A'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B02.tif', 'B02'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B03.tif', 'B03'), + ('HLS.S30.T14SKH.2019211T172909.v2.0.B02.tif', 'B02'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B04.tif', 'B04'), + ('HLS.S30.T14SKH.2019211T172909.v2.0.B03.tif', 'B03'), + ('HLS.S30.T14TLQ.2020156T172859.v2.0.B02.tif', 'B02'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.Fmask.tif', 'Fmask'), +]) +def test_band_id_from_filename_hls_s30(filename, expected_band): + assert models.band_id_from_filename(filename, 'HLS_S30').id == expected_band From 670527b4a68566186828f97e5679450f990c724a Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 31 Mar 2026 15:04:28 -0800 Subject: [PATCH 07/21] merge modality with example notebook --- notebooks/hwds-example.ipynb | 744 ++++++++++++++++++++++++++++++++++ src/satchip/merge_modality.py | 2 + tests/test_merge_modality.py | 4 + 3 files changed, 750 insertions(+) create mode 100644 notebooks/hwds-example.ipynb diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb new file mode 100644 index 0000000..d77d456 --- /dev/null +++ b/notebooks/hwds-example.ipynb @@ -0,0 +1,744 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "4dcd6420", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/wbhorn/miniforge3/envs/satchip/lib/python3.14/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "import geopandas as gpd\n", + "import pandas as pd\n", + "\n", + "from satchip import models, download_data, merge_modality" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "257c1816", + "metadata": {}, + "outputs": [], + "source": [ + "modality = models.HLS_S30\n", + "\n", + "DATA_PATH = Path.cwd() / 'data' \n", + "MODALITY_PATH = DATA_PATH / modality['id']\n", + "\n", + "RAW_DATA_PATH = MODALITY_PATH / 'raw'\n", + "MERGED_PATH = MODALITY_PATH / 'merged'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2b0886a1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SwathDateHLSIDMODISIDVisibilityHLSDategeometry
02018-06-2395a9512018-07-02POLYGON Z ((-97.46296 41.72599 0, -97.45655 41...
12019-07-09102a10222019-07-17POLYGON Z ((-99.49996 40.27048 0, -99.47639 40...
22019-08-13106a10622019-08-28POLYGON Z ((-101.59063 40.78168 0, -101.60341 ...
32019-08-22110a11012019-09-05POLYGON Z ((-98.45623 40.55878 0, -98.45898 40...
42019-08-22110c11022019-09-05POLYGON Z ((-98.45257 40.77528 0, -98.4505 40....
52019-08-13106d10632019-08-28POLYGON Z ((-100.72293 40.03128 0, -100.6802 4...
62019-08-22110g11032019-09-05POLYGON Z ((-98.14395 40.41724 0, -98.14521 40...
72019-08-22110e11032019-09-05POLYGON Z ((-98.40988 40.7443 0, -98.3798 40.7...
82018-08-06126a12612018-08-11POLYGON Z ((-98.2416 41.16581 0, -98.20383 41....
92018-08-06126b12622018-08-11POLYGON Z ((-97.78549 41.00092 0, -97.76367 41...
102018-07-29129a12922018-08-07POLYGON Z ((-103.1513 43.15378 0, -103.14676 4...
112018-07-27130a13012018-08-07POLYGON Z ((-102.98845 43.56544 0, -102.95784 ...
122018-07-27132a13212018-08-11POLYGON Z ((-97.66403 41.83384 0, -97.65714 41...
132018-07-27132b13232018-08-11POLYGON Z ((-97.64508 41.77974 0, -97.62665 41...
142018-07-27132c13232018-08-11POLYGON Z ((-97.47604 41.61431 0, -97.46756 41...
152018-07-18133a13312018-07-26POLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6...
162018-06-30134a13422018-07-12POLYGON Z ((-99.72435 40.41203 0, -99.70477 40...
172018-06-30134e13432018-07-12POLYGON Z ((-99.10277 40.45284 0, -99.07323 40...
182017-07-05597b59712017-07-15POLYGON Z ((-98.60738 45.24939 0, -98.60167 45...
192018-06-26611a61112018-07-05POLYGON Z ((-100.56797 44.73375 0, -100.55907 ...
202018-06-29613b61322018-07-08POLYGON Z ((-100.45872 44.70176 0, -100.42454 ...
212019-08-06623d62322019-08-19MULTIPOLYGON Z (((-100.02775 45.57471 0, -100....
222019-08-06623a62312019-08-19POLYGON Z ((-100.33919 46.03454 0, -100.33567 ...
232018-07-02889a88922018-07-13POLYGON Z ((-102.82547 46.67304 0, -102.82511 ...
242018-07-02889b88912018-07-13POLYGON Z ((-101.87838 46.45158 0, -101.8724 4...
252018-06-26892a89222018-07-08POLYGON Z ((-103.21657 46.25444 0, -103.21434 ...
262017-07-20915a91532017-07-26POLYGON Z ((-102.61561 47.46114 0, -102.55608 ...
272019-07-291079a107912019-08-04POLYGON Z ((-102.59566 39.43458 0, -102.60361 ...
282019-08-131069a106932019-08-14POLYGON Z ((-102.1592 39.1188 0, -102.12104 39...
292018-07-101378a137832018-07-12POLYGON Z ((-95.05656 43.17247 0, -95.04818 43...
302018-06-101380a138032018-06-16POLYGON Z ((-93.91258 40.9999 0, -93.8989 41.0...
312019-07-03628a62832019-07-18POLYGON Z ((-102.31279 43.36328 0, -102.30699 ...
322020-06-04638a63822020-06-12POLYGON Z ((-103.28737 45.58749 0, -103.23569 ...
332020-06-07116d11622020-06-17POLYGON Z ((-101.22083 43.22103 0, -101.2127 4...
342020-06-07116a11632020-06-17POLYGON Z ((-101.44053 42.77567 0, -101.43648 ...
352020-06-07116e11632020-06-17POLYGON Z ((-100.62852 43.61074 0, -100.60716 ...
362020-07-06648a64812020-07-16POLYGON Z ((-97.35114 43.00508 0, -97.33376 43...
372017-08-101052a105232017-08-13POLYGON Z ((-102.68889 38.15458 0, -102.68788 ...
382018-07-261055a105522018-08-04POLYGON Z ((-102.09648 39.57415 0, -102.09107 ...
392018-07-281056a105622018-08-07POLYGON Z ((-102.25777 40.91481 0, -102.26451 ...
402018-07-281056c105632018-08-07POLYGON Z ((-102.0567 40.7458 0, -102.06301 40...
412018-07-281347a134722018-08-04POLYGON Z ((-100.71466 39.28238 0, -100.65799 ...
422018-06-191064a106422018-07-03POLYGON Z ((-102.42594 39.84049 0, -102.42101 ...
432019-08-241338a133832019-09-05POLYGON Z ((-100.22342 39.23137 0, -100.20831 ...
442018-07-271344a134422018-08-02POLYGON Z ((-99.62939 38.06636 0, -99.61494 38...
452020-08-101373a137312020-08-17POLYGON Z ((-94.97885 42.20506 0, -94.97314 42...
462020-08-101373d137332020-08-17POLYGON Z ((-94.80415 42.01749 0, -94.78768 42...
472020-08-101373e137332020-08-17POLYGON Z ((-94.5605 41.97742 0, -94.52217 41....
482020-07-111376a137612020-07-19POLYGON Z ((-92.15179 43.02718 0, -92.1428 43....
\n", + "
" + ], + "text/plain": [ + " SwathDate HLSID MODISID Visibility HLSDate \\\n", + "0 2018-06-23 95a 95 1 2018-07-02 \n", + "1 2019-07-09 102a 102 2 2019-07-17 \n", + "2 2019-08-13 106a 106 2 2019-08-28 \n", + "3 2019-08-22 110a 110 1 2019-09-05 \n", + "4 2019-08-22 110c 110 2 2019-09-05 \n", + "5 2019-08-13 106d 106 3 2019-08-28 \n", + "6 2019-08-22 110g 110 3 2019-09-05 \n", + "7 2019-08-22 110e 110 3 2019-09-05 \n", + "8 2018-08-06 126a 126 1 2018-08-11 \n", + "9 2018-08-06 126b 126 2 2018-08-11 \n", + "10 2018-07-29 129a 129 2 2018-08-07 \n", + "11 2018-07-27 130a 130 1 2018-08-07 \n", + "12 2018-07-27 132a 132 1 2018-08-11 \n", + "13 2018-07-27 132b 132 3 2018-08-11 \n", + "14 2018-07-27 132c 132 3 2018-08-11 \n", + "15 2018-07-18 133a 133 1 2018-07-26 \n", + "16 2018-06-30 134a 134 2 2018-07-12 \n", + "17 2018-06-30 134e 134 3 2018-07-12 \n", + "18 2017-07-05 597b 597 1 2017-07-15 \n", + "19 2018-06-26 611a 611 1 2018-07-05 \n", + "20 2018-06-29 613b 613 2 2018-07-08 \n", + "21 2019-08-06 623d 623 2 2019-08-19 \n", + "22 2019-08-06 623a 623 1 2019-08-19 \n", + "23 2018-07-02 889a 889 2 2018-07-13 \n", + "24 2018-07-02 889b 889 1 2018-07-13 \n", + "25 2018-06-26 892a 892 2 2018-07-08 \n", + "26 2017-07-20 915a 915 3 2017-07-26 \n", + "27 2019-07-29 1079a 1079 1 2019-08-04 \n", + "28 2019-08-13 1069a 1069 3 2019-08-14 \n", + "29 2018-07-10 1378a 1378 3 2018-07-12 \n", + "30 2018-06-10 1380a 1380 3 2018-06-16 \n", + "31 2019-07-03 628a 628 3 2019-07-18 \n", + "32 2020-06-04 638a 638 2 2020-06-12 \n", + "33 2020-06-07 116d 116 2 2020-06-17 \n", + "34 2020-06-07 116a 116 3 2020-06-17 \n", + "35 2020-06-07 116e 116 3 2020-06-17 \n", + "36 2020-07-06 648a 648 1 2020-07-16 \n", + "37 2017-08-10 1052a 1052 3 2017-08-13 \n", + "38 2018-07-26 1055a 1055 2 2018-08-04 \n", + "39 2018-07-28 1056a 1056 2 2018-08-07 \n", + "40 2018-07-28 1056c 1056 3 2018-08-07 \n", + "41 2018-07-28 1347a 1347 2 2018-08-04 \n", + "42 2018-06-19 1064a 1064 2 2018-07-03 \n", + "43 2019-08-24 1338a 1338 3 2019-09-05 \n", + "44 2018-07-27 1344a 1344 2 2018-08-02 \n", + "45 2020-08-10 1373a 1373 1 2020-08-17 \n", + "46 2020-08-10 1373d 1373 3 2020-08-17 \n", + "47 2020-08-10 1373e 1373 3 2020-08-17 \n", + "48 2020-07-11 1376a 1376 1 2020-07-19 \n", + "\n", + " geometry \n", + "0 POLYGON Z ((-97.46296 41.72599 0, -97.45655 41... \n", + "1 POLYGON Z ((-99.49996 40.27048 0, -99.47639 40... \n", + "2 POLYGON Z ((-101.59063 40.78168 0, -101.60341 ... \n", + "3 POLYGON Z ((-98.45623 40.55878 0, -98.45898 40... \n", + "4 POLYGON Z ((-98.45257 40.77528 0, -98.4505 40.... \n", + "5 POLYGON Z ((-100.72293 40.03128 0, -100.6802 4... \n", + "6 POLYGON Z ((-98.14395 40.41724 0, -98.14521 40... \n", + "7 POLYGON Z ((-98.40988 40.7443 0, -98.3798 40.7... \n", + "8 POLYGON Z ((-98.2416 41.16581 0, -98.20383 41.... \n", + "9 POLYGON Z ((-97.78549 41.00092 0, -97.76367 41... \n", + "10 POLYGON Z ((-103.1513 43.15378 0, -103.14676 4... \n", + "11 POLYGON Z ((-102.98845 43.56544 0, -102.95784 ... \n", + "12 POLYGON Z ((-97.66403 41.83384 0, -97.65714 41... \n", + "13 POLYGON Z ((-97.64508 41.77974 0, -97.62665 41... \n", + "14 POLYGON Z ((-97.47604 41.61431 0, -97.46756 41... \n", + "15 POLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6... \n", + "16 POLYGON Z ((-99.72435 40.41203 0, -99.70477 40... \n", + "17 POLYGON Z ((-99.10277 40.45284 0, -99.07323 40... \n", + "18 POLYGON Z ((-98.60738 45.24939 0, -98.60167 45... \n", + "19 POLYGON Z ((-100.56797 44.73375 0, -100.55907 ... \n", + "20 POLYGON Z ((-100.45872 44.70176 0, -100.42454 ... \n", + "21 MULTIPOLYGON Z (((-100.02775 45.57471 0, -100.... \n", + "22 POLYGON Z ((-100.33919 46.03454 0, -100.33567 ... \n", + "23 POLYGON Z ((-102.82547 46.67304 0, -102.82511 ... \n", + "24 POLYGON Z ((-101.87838 46.45158 0, -101.8724 4... \n", + "25 POLYGON Z ((-103.21657 46.25444 0, -103.21434 ... \n", + "26 POLYGON Z ((-102.61561 47.46114 0, -102.55608 ... \n", + "27 POLYGON Z ((-102.59566 39.43458 0, -102.60361 ... \n", + "28 POLYGON Z ((-102.1592 39.1188 0, -102.12104 39... \n", + "29 POLYGON Z ((-95.05656 43.17247 0, -95.04818 43... \n", + "30 POLYGON Z ((-93.91258 40.9999 0, -93.8989 41.0... \n", + "31 POLYGON Z ((-102.31279 43.36328 0, -102.30699 ... \n", + "32 POLYGON Z ((-103.28737 45.58749 0, -103.23569 ... \n", + "33 POLYGON Z ((-101.22083 43.22103 0, -101.2127 4... \n", + "34 POLYGON Z ((-101.44053 42.77567 0, -101.43648 ... \n", + "35 POLYGON Z ((-100.62852 43.61074 0, -100.60716 ... \n", + "36 POLYGON Z ((-97.35114 43.00508 0, -97.33376 43... \n", + "37 POLYGON Z ((-102.68889 38.15458 0, -102.68788 ... \n", + "38 POLYGON Z ((-102.09648 39.57415 0, -102.09107 ... \n", + "39 POLYGON Z ((-102.25777 40.91481 0, -102.26451 ... \n", + "40 POLYGON Z ((-102.0567 40.7458 0, -102.06301 40... \n", + "41 POLYGON Z ((-100.71466 39.28238 0, -100.65799 ... \n", + "42 POLYGON Z ((-102.42594 39.84049 0, -102.42101 ... \n", + "43 POLYGON Z ((-100.22342 39.23137 0, -100.20831 ... \n", + "44 POLYGON Z ((-99.62939 38.06636 0, -99.61494 38... \n", + "45 POLYGON Z ((-94.97885 42.20506 0, -94.97314 42... \n", + "46 POLYGON Z ((-94.80415 42.01749 0, -94.78768 42... \n", + "47 POLYGON Z ((-94.5605 41.97742 0, -94.52217 41.... \n", + "48 POLYGON Z ((-92.15179 43.02718 0, -92.1428 43.... " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "shp_path = \"hwds_pristine\"\n", + "df = gpd.read_file(shp_path)\n", + "\n", + "df[\"SwathDate\"] = pd.to_datetime(df[\"SwathDate\"], format=\"%Y-%m-%d\")\n", + "df[\"HLSDate\"] = pd.to_datetime(df[\"HLSDate\"], format=\"%Y-%m-%d\")\n", + "\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3dc85728", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SwathDate 2019-07-09 00:00:00\n", + "HLSID 102a\n", + "MODISID 102\n", + "Visibility 2\n", + "HLSDate 2019-07-17 00:00:00\n", + "geometry POLYGON Z ((-99.49996223199997 40.270482574000...\n", + "Name: 1, dtype: object" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "swath = df.iloc[1]\n", + "swath" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a611425", + "metadata": {}, + "outputs": [], + "source": [ + "event = models.Event(\n", + " name=swath['HLSID'],\n", + " date=swath['SwathDate'],\n", + " wgs84_geometry=swath['geometry'],\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d0202e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logging in to earthaccess\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11208.06it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 234464.20it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 441505.68it/s]\n" + ] + } + ], + "source": [ + "local_files = download_data.download_data(event, modality, RAW_DATA_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b99ff34", + "metadata": {}, + "outputs": [], + "source": [ + "merged_event = merge_modality.merge_modality(local_files, modality, event=event, output_path=MERGED_PATH)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "satchip", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 74fb26e..ad23634 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -9,6 +9,8 @@ def merge_modality(modality_files: list[Path], modality: models.Modality, event: models.Event, output_path: Path, selected_bands: list[models.Band] | None = None): + output_path.mkdir(exist_ok=True, parents=True) + if len(modality_files) == 0: print(f"Warning: no data for {event.name}") return [] diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 3572bae..9f937a3 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -19,3 +19,7 @@ def test_merge_s2_modality(s2_local_files, s2_event, tmp_path): result = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path) assert result + + result = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') + + assert result From 8076e48b65467f18fffe9b854ce81055e44b254e Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 1 Apr 2026 09:32:46 -0800 Subject: [PATCH 08/21] Add stack_bands function and test --- src/satchip/merge_modality.py | 30 +++++++++++++++++++++++++----- src/satchip/models.py | 4 ++-- src/satchip/mosaic.py | 2 -- tests/test_merge_modality.py | 14 ++++++++++---- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index ad23634..3ada57d 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -1,5 +1,6 @@ -from pathlib import Path import datetime +from collections.abc import Iterable +from pathlib import Path import numpy as np import rasterio @@ -8,18 +9,18 @@ from satchip import models -def merge_modality(modality_files: list[Path], modality: models.Modality, event: models.Event, output_path: Path, selected_bands: list[models.Band] | None = None): +def merge_modality(modality_files: list[Path], modality: models.Modality, event: models.Event, output_path: Path, selected_bands: list[models.Band] | None = None) -> list[Path]: output_path.mkdir(exist_ok=True, parents=True) if len(modality_files) == 0: print(f"Warning: no data for {event.name}") return [] - merged = {} - if selected_bands is None: selected_bands = modality['bands'] + merged = [] + for band in selected_bands: band_files = [f for f in modality_files if band.id in models.band_id_from_filename(f.name, modality['id'])] @@ -29,7 +30,7 @@ def merge_modality(modality_files: list[Path], modality: models.Modality, event: band_files, output_file=output_path / merged_name ) - merged[band] = merged_band_path + merged.append(merged_band_path) return merged @@ -71,3 +72,22 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: ds.close() return output_file + + +def stack_bands(band_files: Iterable[Path], stacked_filename: Path) -> Path: + with rasterio.open(band_files[0]) as src: + meta = src.meta.copy() + + meta.update(count=len(band_files), dtype=np.float32) + + with rasterio.open(stacked_filename, "w", **meta) as dst: + for idx, band_file in enumerate(band_files, start=1): + with rasterio.open(band_file) as src: + + dst.write(src.read(1), idx) + + return stacked_filename + + +def _rename(path: Path, extension: str, mask_name: str) -> Path: + return path.parent / path.name.replace(extension, mask_name) diff --git a/src/satchip/models.py b/src/satchip/models.py index f176b87..47c7aac 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -57,7 +57,7 @@ class ModalityError(Exception): Band("B8A", "NIR Narrow", "N"), Band("B11", "SWIR 1", "SW1"), Band("B12", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "Fmask"), + Band("Fmask", "Cloud Mask", "fmask"), ) }, "HLS_L30": { @@ -70,7 +70,7 @@ class ModalityError(Exception): Band("B05", "NIR Narrow", "N"), Band("B06", "SWIR 1", "SW1"), Band("B07", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "Fmask"), + Band("Fmask", "Cloud Mask", "fmask"), ) } } diff --git a/src/satchip/mosaic.py b/src/satchip/mosaic.py index 21e4eac..f42d73c 100644 --- a/src/satchip/mosaic.py +++ b/src/satchip/mosaic.py @@ -10,8 +10,6 @@ from rasterio.warp import calculate_default_transform, reproject from rasterio.warp import Resampling, transform_bounds from rasterio.transform import from_bounds -from rasterio.mask import mask -from shapely.geometry import box from modality import Modality import hls diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 9f937a3..0985ced 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -16,10 +16,16 @@ def test_merge_s2_modality_empty(s2_local_files, s2_event, tmp_path): def test_merge_s2_modality(s2_local_files, s2_event, tmp_path): - result = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path) + merged_files = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') - assert result + assert len(merged_files) == len(models.HLS_S30['bands']) + assert all('merged' in str(result) for result in merged_files) - result = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') - assert result +def test_stack_bands(s2_local_files, s2_event, tmp_path): + bands = tuple(b for b in models.HLS_S30['bands'] if b.shortname != 'fmask') + + merged_bands = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged', bands) + stacked_data = merge_modality.stack_bands(merged_bands, stacked_filename=tmp_path / 'stacked.tif') + + assert 'stacked.tif' in stacked_data.name From 0cf904e9a20b31e7e60294227c555961d83448a6 Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 1 Apr 2026 10:24:39 -0800 Subject: [PATCH 09/21] Make parent dir when stacking bands --- src/satchip/merge_modality.py | 2 ++ src/satchip/models.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 3ada57d..cbb98b0 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -75,6 +75,8 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: def stack_bands(band_files: Iterable[Path], stacked_filename: Path) -> Path: + stacked_filename.parent.mkdir(exist_ok=True, parents=True) + with rasterio.open(band_files[0]) as src: meta = src.meta.copy() diff --git a/src/satchip/models.py b/src/satchip/models.py index 47c7aac..11f65ed 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -12,14 +12,6 @@ class Band(NamedTuple): shortname: str -def bands_by_shortname(bands: tuple[Band, ...]) -> dict[str, Band]: - return {band.shortname: band for band in bands} - - -def bands_by_id(bands: tuple[Band, ...]) -> dict[str, Band]: - return {band.id: band for band in bands} - - class Modality(TypedDict): id: str bands: Tuple[Band, ...] @@ -37,6 +29,14 @@ class ModalityError(Exception): pass +def bands_by_shortname(bands: tuple[Band, ...]) -> dict[str, Band]: + return {band.shortname: band for band in bands} + + +def bands_by_id(bands: tuple[Band, ...]) -> dict[str, Band]: + return {band.id: band for band in bands} + + MODALITIES: dict[str, Modality] = { "OPERA_RTC": { "id": "OPERA_RTC", @@ -57,7 +57,7 @@ class ModalityError(Exception): Band("B8A", "NIR Narrow", "N"), Band("B11", "SWIR 1", "SW1"), Band("B12", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "fmask"), + Band("Fmask", "Cloud Mask", "Fmask"), ) }, "HLS_L30": { From 0169313ae3e0bcc49bad0eaae98bd51afd75d95d Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 1 Apr 2026 11:03:35 -0800 Subject: [PATCH 10/21] Add better tests for merge modality --- tests/test_merge_modality.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 0985ced..21e73dd 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,4 +1,5 @@ from satchip import merge_modality, models +import rasterio def test_make_merge_name(s2_event): @@ -21,11 +22,23 @@ def test_merge_s2_modality(s2_local_files, s2_event, tmp_path): assert len(merged_files) == len(models.HLS_S30['bands']) assert all('merged' in str(result) for result in merged_files) + shapes = set() + for merged_file in merged_files: + with rasterio.open(merged_file) as ds: + shapes.add(ds.shape) + + assert len(shapes) == 1 + def test_stack_bands(s2_local_files, s2_event, tmp_path): - bands = tuple(b for b in models.HLS_S30['bands'] if b.shortname != 'fmask') + bands = tuple(b for b in models.HLS_S30['bands'] if b.id != 'Fmask') merged_bands = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged', bands) stacked_data = merge_modality.stack_bands(merged_bands, stacked_filename=tmp_path / 'stacked.tif') assert 'stacked.tif' in stacked_data.name + + with rasterio.open(stacked_data) as ds: + num_bands = ds.count + + assert num_bands == len(bands) From 7acebc44d8a42829805a751270e4764504f3dba0 Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 2 Apr 2026 13:07:00 -0800 Subject: [PATCH 11/21] Add reproj, and generate labels functions --- src/satchip/generate_labels.py | 32 ++++++++++++++++++++ src/satchip/merge_modality.py | 53 +++++++++++++++++++++++++++++++++- src/satchip/models.py | 6 ++-- tests/test_generate_labels.py | 15 ++++++++++ tests/test_merge_modality.py | 11 +++++++ 5 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 src/satchip/generate_labels.py create mode 100644 tests/test_generate_labels.py diff --git a/src/satchip/generate_labels.py b/src/satchip/generate_labels.py new file mode 100644 index 0000000..6c5cf55 --- /dev/null +++ b/src/satchip/generate_labels.py @@ -0,0 +1,32 @@ +from pathlib import Path + +import rasterio +from rasterio import features + +from satchip import models + + +def binary_mask_from_template( + template_data_path: Path, event: models.Event, output_dir: Path +) -> tuple[Path, Path]: + output_dir.mkdir(exist_ok=True, parents=True) + + mask_path = output_dir / f"{event.name}.MASK.tif" + + with rasterio.open(template_data_path) as ds: + profile = ds.profile + + mask_raster = features.rasterize( + shapes=[[event.wgs84_geometry, 1]], + fill=0, + out_shape=ds.shape, + transform=ds.transform, + ) + + profile.update(dtype="uint8", count=1, nodata=255) + + with rasterio.open(mask_path, "w", **profile) as dst: + dst.write(mask_raster, 1) + print("generated:", mask_path) + + return mask_path diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index cbb98b0..d87cdf6 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -4,12 +4,20 @@ import numpy as np import rasterio +from rasterio.crs import CRS +from rasterio.warp import calculate_default_transform, reproject from rasterio.merge import merge from satchip import models -def merge_modality(modality_files: list[Path], modality: models.Modality, event: models.Event, output_path: Path, selected_bands: list[models.Band] | None = None) -> list[Path]: +def merge_modality( + modality_files: list[Path], + modality: models.Modality, + event: models.Event, + output_path: Path, + selected_bands: list[models.Band] | None = None +) -> list[Path]: output_path.mkdir(exist_ok=True, parents=True) if len(modality_files) == 0: @@ -91,5 +99,48 @@ def stack_bands(band_files: Iterable[Path], stacked_filename: Path) -> Path: return stacked_filename +def reproject_files( + files: list[Path], output_dir: Path +) -> list[Path]: + output_dir.mkdir(exist_ok=True, parents=True) + + reprojected_paths = [ + output_dir / f"{file.name}" for file in files + ] + + for file, output in zip(files, reprojected_paths): + if output.exists(): + continue + + print(f"reprojecting to wgs84: {output.name}") + reproject_file(file, output) + + return reprojected_paths + + +def reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: + # https://rasterio.readthedocs.io/en/stable/topics/reproject.html#reprojecting-a-geotiff-dataset + with rasterio.open(local_file) as src: + dst_crs = CRS.from_epsg(epsg) + transform, width, height = calculate_default_transform( + src.crs, dst_crs, src.width, src.height, *src.bounds + ) + + dst_kwargs = src.meta.copy() + dst_kwargs.update( + {"crs": dst_crs, "transform": transform, "width": width, "height": height} + ) + + with rasterio.open(reprojected_file, "w", **dst_kwargs) as dst: + for i in range(1, src.count + 1): + reproject( + source=rasterio.band(src, i), + destination=rasterio.band(dst, i), + src_transform=src.transform, + src_crs=src.crs, + dst_transform=transform, + dst_crs=dst_crs, + ) + def _rename(path: Path, extension: str, mask_name: str) -> Path: return path.parent / path.name.replace(extension, mask_name) diff --git a/src/satchip/models.py b/src/satchip/models.py index 11f65ed..9a5e955 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -90,9 +90,11 @@ def bands_by_id(bands: tuple[Band, ...]) -> dict[str, Band]: def band_id_from_filename(filename: str, modality_id: str) -> Band | None: - if 'HLS' in modality_id: + if 'HLS_L30' in modality_id: sensor, band = band_from_hls_filename(filename) - elif 'RTC' in modality_id: + elif 'HLS_S30' in modality_id: + sensor, band = band_from_hls_filename(filename) + elif 'OPERA_RTC' in modality_id: sensor, band = band_from_rtc_filename(filename) else: raise ModalityError(f'Modality not found {modality_id}, must be ({MODALITY_IDS})') diff --git a/tests/test_generate_labels.py b/tests/test_generate_labels.py new file mode 100644 index 0000000..95be506 --- /dev/null +++ b/tests/test_generate_labels.py @@ -0,0 +1,15 @@ +import rasterio + +from satchip import generate_labels + + +def test_generate_labales(s2_local_files, s2_event, tmp_path): + template_file = s2_local_files[0] + + output = generate_labels.binary_mask_from_template(template_file, s2_event, tmp_path / 'labels') + + assert output.name.endswith('MASK.tif') + + with rasterio.open(output) as mask_ds: + with rasterio.open(template_file) as template_ds: + assert mask_ds.shape == template_ds.shape diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 21e73dd..1693950 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -42,3 +42,14 @@ def test_stack_bands(s2_local_files, s2_event, tmp_path): num_bands = ds.count assert num_bands == len(bands) + + +def test_reproject_files(s2_local_files, s2_event, tmp_path): + reprojected_files = merge_modality.reproject_files(s2_local_files, tmp_path / 'wgs84') + + assert len(reprojected_files) == len(s2_local_files) + + for f in reprojected_files: + with rasterio.open(f) as ds: + epsg_code = ds.profile['crs'].to_epsg() + assert epsg_code == 4326 From c7f9f9c09137cce1bea5ae3cb738d758a7928545 Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 2 Apr 2026 15:23:15 -0800 Subject: [PATCH 12/21] Add function for warping files to reference --- notebooks/hwds-example.ipynb | 735 +++++++--------------------------- src/satchip/merge_modality.py | 83 +++- tests/test_merge_modality.py | 23 +- 3 files changed, 238 insertions(+), 603 deletions(-) diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index d77d456..853dcc5 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "4dcd6420", "metadata": {}, "outputs": [ @@ -21,12 +21,12 @@ "import geopandas as gpd\n", "import pandas as pd\n", "\n", - "from satchip import models, download_data, merge_modality" + "from satchip import models, download_data, merge_modality, generate_labels" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "257c1816", "metadata": {}, "outputs": [], @@ -37,7 +37,10 @@ "MODALITY_PATH = DATA_PATH / modality['id']\n", "\n", "RAW_DATA_PATH = MODALITY_PATH / 'raw'\n", - "MERGED_PATH = MODALITY_PATH / 'merged'" + "MERGED_PATH = MODALITY_PATH / 'merged'\n", + "REPROJECTED_PATH = MODALITY_PATH / 'wgs84'\n", + "STACKED_PATH = MODALITY_PATH / 'stacked'\n", + "WARPED_PATH = MODALITY_PATH / 'warped' " ] }, { @@ -45,599 +48,13 @@ "execution_count": 3, "id": "2b0886a1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
SwathDateHLSIDMODISIDVisibilityHLSDategeometry
02018-06-2395a9512018-07-02POLYGON Z ((-97.46296 41.72599 0, -97.45655 41...
12019-07-09102a10222019-07-17POLYGON Z ((-99.49996 40.27048 0, -99.47639 40...
22019-08-13106a10622019-08-28POLYGON Z ((-101.59063 40.78168 0, -101.60341 ...
32019-08-22110a11012019-09-05POLYGON Z ((-98.45623 40.55878 0, -98.45898 40...
42019-08-22110c11022019-09-05POLYGON Z ((-98.45257 40.77528 0, -98.4505 40....
52019-08-13106d10632019-08-28POLYGON Z ((-100.72293 40.03128 0, -100.6802 4...
62019-08-22110g11032019-09-05POLYGON Z ((-98.14395 40.41724 0, -98.14521 40...
72019-08-22110e11032019-09-05POLYGON Z ((-98.40988 40.7443 0, -98.3798 40.7...
82018-08-06126a12612018-08-11POLYGON Z ((-98.2416 41.16581 0, -98.20383 41....
92018-08-06126b12622018-08-11POLYGON Z ((-97.78549 41.00092 0, -97.76367 41...
102018-07-29129a12922018-08-07POLYGON Z ((-103.1513 43.15378 0, -103.14676 4...
112018-07-27130a13012018-08-07POLYGON Z ((-102.98845 43.56544 0, -102.95784 ...
122018-07-27132a13212018-08-11POLYGON Z ((-97.66403 41.83384 0, -97.65714 41...
132018-07-27132b13232018-08-11POLYGON Z ((-97.64508 41.77974 0, -97.62665 41...
142018-07-27132c13232018-08-11POLYGON Z ((-97.47604 41.61431 0, -97.46756 41...
152018-07-18133a13312018-07-26POLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6...
162018-06-30134a13422018-07-12POLYGON Z ((-99.72435 40.41203 0, -99.70477 40...
172018-06-30134e13432018-07-12POLYGON Z ((-99.10277 40.45284 0, -99.07323 40...
182017-07-05597b59712017-07-15POLYGON Z ((-98.60738 45.24939 0, -98.60167 45...
192018-06-26611a61112018-07-05POLYGON Z ((-100.56797 44.73375 0, -100.55907 ...
202018-06-29613b61322018-07-08POLYGON Z ((-100.45872 44.70176 0, -100.42454 ...
212019-08-06623d62322019-08-19MULTIPOLYGON Z (((-100.02775 45.57471 0, -100....
222019-08-06623a62312019-08-19POLYGON Z ((-100.33919 46.03454 0, -100.33567 ...
232018-07-02889a88922018-07-13POLYGON Z ((-102.82547 46.67304 0, -102.82511 ...
242018-07-02889b88912018-07-13POLYGON Z ((-101.87838 46.45158 0, -101.8724 4...
252018-06-26892a89222018-07-08POLYGON Z ((-103.21657 46.25444 0, -103.21434 ...
262017-07-20915a91532017-07-26POLYGON Z ((-102.61561 47.46114 0, -102.55608 ...
272019-07-291079a107912019-08-04POLYGON Z ((-102.59566 39.43458 0, -102.60361 ...
282019-08-131069a106932019-08-14POLYGON Z ((-102.1592 39.1188 0, -102.12104 39...
292018-07-101378a137832018-07-12POLYGON Z ((-95.05656 43.17247 0, -95.04818 43...
302018-06-101380a138032018-06-16POLYGON Z ((-93.91258 40.9999 0, -93.8989 41.0...
312019-07-03628a62832019-07-18POLYGON Z ((-102.31279 43.36328 0, -102.30699 ...
322020-06-04638a63822020-06-12POLYGON Z ((-103.28737 45.58749 0, -103.23569 ...
332020-06-07116d11622020-06-17POLYGON Z ((-101.22083 43.22103 0, -101.2127 4...
342020-06-07116a11632020-06-17POLYGON Z ((-101.44053 42.77567 0, -101.43648 ...
352020-06-07116e11632020-06-17POLYGON Z ((-100.62852 43.61074 0, -100.60716 ...
362020-07-06648a64812020-07-16POLYGON Z ((-97.35114 43.00508 0, -97.33376 43...
372017-08-101052a105232017-08-13POLYGON Z ((-102.68889 38.15458 0, -102.68788 ...
382018-07-261055a105522018-08-04POLYGON Z ((-102.09648 39.57415 0, -102.09107 ...
392018-07-281056a105622018-08-07POLYGON Z ((-102.25777 40.91481 0, -102.26451 ...
402018-07-281056c105632018-08-07POLYGON Z ((-102.0567 40.7458 0, -102.06301 40...
412018-07-281347a134722018-08-04POLYGON Z ((-100.71466 39.28238 0, -100.65799 ...
422018-06-191064a106422018-07-03POLYGON Z ((-102.42594 39.84049 0, -102.42101 ...
432019-08-241338a133832019-09-05POLYGON Z ((-100.22342 39.23137 0, -100.20831 ...
442018-07-271344a134422018-08-02POLYGON Z ((-99.62939 38.06636 0, -99.61494 38...
452020-08-101373a137312020-08-17POLYGON Z ((-94.97885 42.20506 0, -94.97314 42...
462020-08-101373d137332020-08-17POLYGON Z ((-94.80415 42.01749 0, -94.78768 42...
472020-08-101373e137332020-08-17POLYGON Z ((-94.5605 41.97742 0, -94.52217 41....
482020-07-111376a137612020-07-19POLYGON Z ((-92.15179 43.02718 0, -92.1428 43....
\n", - "
" - ], - "text/plain": [ - " SwathDate HLSID MODISID Visibility HLSDate \\\n", - "0 2018-06-23 95a 95 1 2018-07-02 \n", - "1 2019-07-09 102a 102 2 2019-07-17 \n", - "2 2019-08-13 106a 106 2 2019-08-28 \n", - "3 2019-08-22 110a 110 1 2019-09-05 \n", - "4 2019-08-22 110c 110 2 2019-09-05 \n", - "5 2019-08-13 106d 106 3 2019-08-28 \n", - "6 2019-08-22 110g 110 3 2019-09-05 \n", - "7 2019-08-22 110e 110 3 2019-09-05 \n", - "8 2018-08-06 126a 126 1 2018-08-11 \n", - "9 2018-08-06 126b 126 2 2018-08-11 \n", - "10 2018-07-29 129a 129 2 2018-08-07 \n", - "11 2018-07-27 130a 130 1 2018-08-07 \n", - "12 2018-07-27 132a 132 1 2018-08-11 \n", - "13 2018-07-27 132b 132 3 2018-08-11 \n", - "14 2018-07-27 132c 132 3 2018-08-11 \n", - "15 2018-07-18 133a 133 1 2018-07-26 \n", - "16 2018-06-30 134a 134 2 2018-07-12 \n", - "17 2018-06-30 134e 134 3 2018-07-12 \n", - "18 2017-07-05 597b 597 1 2017-07-15 \n", - "19 2018-06-26 611a 611 1 2018-07-05 \n", - "20 2018-06-29 613b 613 2 2018-07-08 \n", - "21 2019-08-06 623d 623 2 2019-08-19 \n", - "22 2019-08-06 623a 623 1 2019-08-19 \n", - "23 2018-07-02 889a 889 2 2018-07-13 \n", - "24 2018-07-02 889b 889 1 2018-07-13 \n", - "25 2018-06-26 892a 892 2 2018-07-08 \n", - "26 2017-07-20 915a 915 3 2017-07-26 \n", - "27 2019-07-29 1079a 1079 1 2019-08-04 \n", - "28 2019-08-13 1069a 1069 3 2019-08-14 \n", - "29 2018-07-10 1378a 1378 3 2018-07-12 \n", - "30 2018-06-10 1380a 1380 3 2018-06-16 \n", - "31 2019-07-03 628a 628 3 2019-07-18 \n", - "32 2020-06-04 638a 638 2 2020-06-12 \n", - "33 2020-06-07 116d 116 2 2020-06-17 \n", - "34 2020-06-07 116a 116 3 2020-06-17 \n", - "35 2020-06-07 116e 116 3 2020-06-17 \n", - "36 2020-07-06 648a 648 1 2020-07-16 \n", - "37 2017-08-10 1052a 1052 3 2017-08-13 \n", - "38 2018-07-26 1055a 1055 2 2018-08-04 \n", - "39 2018-07-28 1056a 1056 2 2018-08-07 \n", - "40 2018-07-28 1056c 1056 3 2018-08-07 \n", - "41 2018-07-28 1347a 1347 2 2018-08-04 \n", - "42 2018-06-19 1064a 1064 2 2018-07-03 \n", - "43 2019-08-24 1338a 1338 3 2019-09-05 \n", - "44 2018-07-27 1344a 1344 2 2018-08-02 \n", - "45 2020-08-10 1373a 1373 1 2020-08-17 \n", - "46 2020-08-10 1373d 1373 3 2020-08-17 \n", - "47 2020-08-10 1373e 1373 3 2020-08-17 \n", - "48 2020-07-11 1376a 1376 1 2020-07-19 \n", - "\n", - " geometry \n", - "0 POLYGON Z ((-97.46296 41.72599 0, -97.45655 41... \n", - "1 POLYGON Z ((-99.49996 40.27048 0, -99.47639 40... \n", - "2 POLYGON Z ((-101.59063 40.78168 0, -101.60341 ... \n", - "3 POLYGON Z ((-98.45623 40.55878 0, -98.45898 40... \n", - "4 POLYGON Z ((-98.45257 40.77528 0, -98.4505 40.... \n", - "5 POLYGON Z ((-100.72293 40.03128 0, -100.6802 4... \n", - "6 POLYGON Z ((-98.14395 40.41724 0, -98.14521 40... \n", - "7 POLYGON Z ((-98.40988 40.7443 0, -98.3798 40.7... \n", - "8 POLYGON Z ((-98.2416 41.16581 0, -98.20383 41.... \n", - "9 POLYGON Z ((-97.78549 41.00092 0, -97.76367 41... \n", - "10 POLYGON Z ((-103.1513 43.15378 0, -103.14676 4... \n", - "11 POLYGON Z ((-102.98845 43.56544 0, -102.95784 ... \n", - "12 POLYGON Z ((-97.66403 41.83384 0, -97.65714 41... \n", - "13 POLYGON Z ((-97.64508 41.77974 0, -97.62665 41... \n", - "14 POLYGON Z ((-97.47604 41.61431 0, -97.46756 41... \n", - "15 POLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6... \n", - "16 POLYGON Z ((-99.72435 40.41203 0, -99.70477 40... \n", - "17 POLYGON Z ((-99.10277 40.45284 0, -99.07323 40... \n", - "18 POLYGON Z ((-98.60738 45.24939 0, -98.60167 45... \n", - "19 POLYGON Z ((-100.56797 44.73375 0, -100.55907 ... \n", - "20 POLYGON Z ((-100.45872 44.70176 0, -100.42454 ... \n", - "21 MULTIPOLYGON Z (((-100.02775 45.57471 0, -100.... \n", - "22 POLYGON Z ((-100.33919 46.03454 0, -100.33567 ... \n", - "23 POLYGON Z ((-102.82547 46.67304 0, -102.82511 ... \n", - "24 POLYGON Z ((-101.87838 46.45158 0, -101.8724 4... \n", - "25 POLYGON Z ((-103.21657 46.25444 0, -103.21434 ... \n", - "26 POLYGON Z ((-102.61561 47.46114 0, -102.55608 ... \n", - "27 POLYGON Z ((-102.59566 39.43458 0, -102.60361 ... \n", - "28 POLYGON Z ((-102.1592 39.1188 0, -102.12104 39... \n", - "29 POLYGON Z ((-95.05656 43.17247 0, -95.04818 43... \n", - "30 POLYGON Z ((-93.91258 40.9999 0, -93.8989 41.0... \n", - "31 POLYGON Z ((-102.31279 43.36328 0, -102.30699 ... \n", - "32 POLYGON Z ((-103.28737 45.58749 0, -103.23569 ... \n", - "33 POLYGON Z ((-101.22083 43.22103 0, -101.2127 4... \n", - "34 POLYGON Z ((-101.44053 42.77567 0, -101.43648 ... \n", - "35 POLYGON Z ((-100.62852 43.61074 0, -100.60716 ... \n", - "36 POLYGON Z ((-97.35114 43.00508 0, -97.33376 43... \n", - "37 POLYGON Z ((-102.68889 38.15458 0, -102.68788 ... \n", - "38 POLYGON Z ((-102.09648 39.57415 0, -102.09107 ... \n", - "39 POLYGON Z ((-102.25777 40.91481 0, -102.26451 ... \n", - "40 POLYGON Z ((-102.0567 40.7458 0, -102.06301 40... \n", - "41 POLYGON Z ((-100.71466 39.28238 0, -100.65799 ... \n", - "42 POLYGON Z ((-102.42594 39.84049 0, -102.42101 ... \n", - "43 POLYGON Z ((-100.22342 39.23137 0, -100.20831 ... \n", - "44 POLYGON Z ((-99.62939 38.06636 0, -99.61494 38... \n", - "45 POLYGON Z ((-94.97885 42.20506 0, -94.97314 42... \n", - "46 POLYGON Z ((-94.80415 42.01749 0, -94.78768 42... \n", - "47 POLYGON Z ((-94.5605 41.97742 0, -94.52217 41.... \n", - "48 POLYGON Z ((-92.15179 43.02718 0, -92.1428 43.... " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "shp_path = \"hwds_pristine\"\n", "df = gpd.read_file(shp_path)\n", "\n", "df[\"SwathDate\"] = pd.to_datetime(df[\"SwathDate\"], format=\"%Y-%m-%d\")\n", - "df[\"HLSDate\"] = pd.to_datetime(df[\"HLSDate\"], format=\"%Y-%m-%d\")\n", - "\n", - "df" + "df[\"HLSDate\"] = pd.to_datetime(df[\"HLSDate\"], format=\"%Y-%m-%d\")" ] }, { @@ -670,21 +87,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "6a611425", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Event(name='102a', date=Timestamp('2019-07-09 00:00:00'), wgs84_geometry=)\n" + ] + } + ], "source": [ "event = models.Event(\n", " name=swath['HLSID'],\n", " date=swath['SwathDate'],\n", " wgs84_geometry=swath['geometry'],\n", - ")\n" + ")\n", + "print(event)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "0d0202e9", "metadata": {}, "outputs": [ @@ -699,9 +125,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11208.06it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 234464.20it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 441505.68it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10439.36it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 191617.95it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 424143.10it/s]\n" ] } ], @@ -711,13 +137,122 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "9b99ff34", "metadata": {}, "outputs": [], "source": [ "merged_event = merge_modality.merge_modality(local_files, modality, event=event, output_path=MERGED_PATH)" ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e6ce6b53", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.B.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.G.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.R.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.N.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW1.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW2.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.Fmask.tif')]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "merged_event" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9673d689", + "metadata": {}, + "outputs": [], + "source": [ + "stacked_filename = MERGED_PATH / f'{swath[\"HLSID\"]}.stacked.tif'\n", + "data_bands, fmask = merged_event[:-1], merged_event[-1] \n", + "\n", + "stacked = merge_modality.stack_bands(data_bands, stacked_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b4099849", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.B.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.G.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.R.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.N.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW1.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW2.tif')]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_bands" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e6f7e30b", + "metadata": {}, + "outputs": [], + "source": [ + "stacked, fmask = merge_modality.reproject_files([stacked, fmask], REPROJECTED_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9d9b6664", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/102a.MASK.tif\n" + ] + } + ], + "source": [ + "label = generate_labels.binary_mask_from_template(stacked, event, REPROJECTED_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "3180399a", + "metadata": {}, + "outputs": [], + "source": [ + "warped = merge_modality.warp_to_reference(\n", + " reference_path=label, \n", + " data_files=[stacked, fmask], \n", + " output_dir=WARPED_PATH, \n", + " bounding_box_4326=event.wgs84_geometry.bounds\n", + ")" + ] } ], "metadata": { diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index d87cdf6..916e777 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -5,8 +5,10 @@ import numpy as np import rasterio from rasterio.crs import CRS +from rasterio.warp import Resampling, transform_bounds from rasterio.warp import calculate_default_transform, reproject from rasterio.merge import merge +from rasterio.transform import from_bounds from satchip import models @@ -142,5 +144,82 @@ def reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: dst_crs=dst_crs, ) -def _rename(path: Path, extension: str, mask_name: str) -> Path: - return path.parent / path.name.replace(extension, mask_name) + +def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_dir: Path, bounding_box_4326: tuple(float, float, float, float)): + output_dir.mkdir(parents=True, exist_ok=True) + + dst_transform, width, height, dst_crs = _build_common_grid( + bounding_box_4326, reference_path + ) + + files = (reference_path, *data_files) + + output = [] + for data_file in files: + out_path = output_dir / data_file.name + + _warp_single( + data_file, out_path, + dst_transform, width, height, dst_crs, + ) + + output.append(out_path) + + return output + + +def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, resampling=Resampling.bilinear): + with rasterio.open(input_path) as src: + dst_data = np.zeros((src.count, height, width), dtype=src.dtypes[0]) + + reproject( + source=rasterio.band(src, list(range(1, src.count + 1))), + destination=dst_data, + src_transform=src.transform, + src_crs=src.crs, + dst_transform=dst_transform, + dst_crs=dst_crs, + resampling=resampling, + dst_nodata=src.nodata, + ) + + out_meta = src.meta.copy() + out_meta.update({ + "driver": "GTiff", + "height": height, + "width": width, + "transform": dst_transform, + "crs": dst_crs, + }) + + with rasterio.open(output_path, "w", **out_meta) as dest: + dest.write(dst_data) + + return output_path + + +def _build_common_grid(bounding_box_4326, reference_path): + dst_crs = CRS.from_epsg(4326) + minx, miny, maxx, maxy = bounding_box_4326 + + with rasterio.open(reference_path) as ref: + bounds_4326 = transform_bounds(ref.crs, dst_crs, *ref.bounds, densify_pts=21) + ref_width = ref.width + ref_height = ref.height + + ref_bbox_width = bounds_4326[2] - bounds_4326[0] + ref_bbox_height = bounds_4326[3] - bounds_4326[1] + res_x = ref_bbox_width / ref_width + res_y = ref_bbox_height / ref_height + + minx = np.floor(minx / res_x) * res_x + miny = np.floor(miny / res_y) * res_y + maxx = np.ceil(maxx / res_x) * res_x + maxy = np.ceil(maxy / res_y) * res_y + + width = int(round((maxx - minx) / res_x)) + height = int(round((maxy - miny) / res_y)) + + dst_transform = from_bounds(minx, miny, maxx, maxy, width, height) + + return dst_transform, width, height, dst_crs diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 1693950..8689a6d 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,4 +1,4 @@ -from satchip import merge_modality, models +from satchip import merge_modality, models, generate_labels import rasterio @@ -44,6 +44,27 @@ def test_stack_bands(s2_local_files, s2_event, tmp_path): assert num_bands == len(bands) +def test_warp_to_reference(s2_local_files, s2_event, tmp_path): + merged = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') + reprojected = merge_modality.reproject_files(merged, tmp_path / 'wgs84') + + bands = reprojected[:-1] + fmask = reprojected[-1] + + stacked = merge_modality.stack_bands(bands, stacked_filename=tmp_path / 'stacked.tif') + label = generate_labels.binary_mask_from_template(stacked, s2_event, tmp_path) + + outputs = merge_modality.warp_to_reference(label, [stacked, fmask], tmp_path / 'warped', s2_event.wgs84_geometry.bounds) + + shapes = set() + for output in outputs: + with rasterio.open(output) as ds: + shapes.add(ds.shape) + + assert len(outputs) == 3 + assert len(shapes) == 1 + + def test_reproject_files(s2_local_files, s2_event, tmp_path): reprojected_files = merge_modality.reproject_files(s2_local_files, tmp_path / 'wgs84') From cd831be54338138358b0d2be003adb72cc63ffac Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 3 Apr 2026 09:57:46 -0800 Subject: [PATCH 13/21] Add buffer_m to events --- notebooks/hwds-example.ipynb | 3 ++- src/satchip/download_data.py | 2 +- src/satchip/merge_modality.py | 10 +++++----- src/satchip/models.py | 13 +++++++++++++ tests/test_merge_modality.py | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index 853dcc5..e9fe7f0 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "6a611425", "metadata": {}, "outputs": [ @@ -104,6 +104,7 @@ " name=swath['HLSID'],\n", " date=swath['SwathDate'],\n", " wgs84_geometry=swath['geometry'],\n", + " buffer_m=10000\n", ")\n", "print(event)" ] diff --git a/src/satchip/download_data.py b/src/satchip/download_data.py index cb646c4..906f7c3 100644 --- a/src/satchip/download_data.py +++ b/src/satchip/download_data.py @@ -32,7 +32,7 @@ def _search_data(event: models.Event, modality: models.Modality) -> list[earthac results = earthaccess.search_data( short_name=[collection_id], temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), - bounding_box=event.wgs84_geometry.bounds, + bounding_box=event.buffered_geometry().bounds, ) return results diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 916e777..c030544 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -8,7 +8,7 @@ from rasterio.warp import Resampling, transform_bounds from rasterio.warp import calculate_default_transform, reproject from rasterio.merge import merge -from rasterio.transform import from_bounds +from rasterio.transform import from_bounds, Affine from satchip import models @@ -145,11 +145,11 @@ def reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: ) -def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_dir: Path, bounding_box_4326: tuple(float, float, float, float)): +def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_dir: Path, bounding_box_wgs84: tuple(float, float, float, float)) -> list[Path]: output_dir.mkdir(parents=True, exist_ok=True) dst_transform, width, height, dst_crs = _build_common_grid( - bounding_box_4326, reference_path + bounding_box_wgs84, reference_path ) files = (reference_path, *data_files) @@ -168,7 +168,7 @@ def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_d return output -def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, resampling=Resampling.bilinear): +def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, resampling=Resampling.bilinear) -> Path: with rasterio.open(input_path) as src: dst_data = np.zeros((src.count, height, width), dtype=src.dtypes[0]) @@ -198,7 +198,7 @@ def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, return output_path -def _build_common_grid(bounding_box_4326, reference_path): +def _build_common_grid(bounding_box_4326: tuple(float, float, float, float), reference_path: Path) -> tuple[Affine, int, int, CRS]: dst_crs = CRS.from_epsg(4326) minx, miny, maxx, maxy = bounding_box_4326 diff --git a/src/satchip/models.py b/src/satchip/models.py index 9a5e955..f75049c 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -4,6 +4,7 @@ from typing import Tuple import shapely +import pyproj class Band(NamedTuple): @@ -23,6 +24,18 @@ class Event: name: str date: datetime wgs84_geometry: shapely.geometry.Polygon + buffer_m: int = 0 + + def buffered_geometry(self) -> shapely.geometry.Polygon: + if self.buffer_m == 0: + return self.wgs84_geometry + + to_utm = pyproj.Transformer.from_crs(4326, 32615, always_xy=True) + projected = shapely.ops.transform(to_utm.transform, self.wgs84_geometry) + buffered = projected.buffer(self.buffer_m) + + to_wgs84 = pyproj.Transformer.from_crs(32615, 4326, always_xy=True) + return shapely.ops.transform(to_wgs84.transform, buffered) class ModalityError(Exception): diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 8689a6d..9e9a10c 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -54,7 +54,7 @@ def test_warp_to_reference(s2_local_files, s2_event, tmp_path): stacked = merge_modality.stack_bands(bands, stacked_filename=tmp_path / 'stacked.tif') label = generate_labels.binary_mask_from_template(stacked, s2_event, tmp_path) - outputs = merge_modality.warp_to_reference(label, [stacked, fmask], tmp_path / 'warped', s2_event.wgs84_geometry.bounds) + outputs = merge_modality.warp_to_reference(label, [stacked, fmask], tmp_path / 'warped', s2_event.buffered_geometry().bounds) shapes = set() for output in outputs: From b5c4d5fb4dd8dc2b56ebdd4f4c12b3422f58f735 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 3 Apr 2026 14:00:53 -0800 Subject: [PATCH 14/21] Function to create a grid to chip data on --- src/satchip/chip_data.py | 153 +++++++++++++++++++++++++++++++++++++++ src/satchip/models.py | 23 ++++++ tests/conftest.py | 11 +++ tests/test_chip_data.py | 19 +++++ 4 files changed, 206 insertions(+) create mode 100644 src/satchip/chip_data.py create mode 100644 tests/test_chip_data.py diff --git a/src/satchip/chip_data.py b/src/satchip/chip_data.py new file mode 100644 index 0000000..58e4d5e --- /dev/null +++ b/src/satchip/chip_data.py @@ -0,0 +1,153 @@ +from pathlib import Path +from collections.abc import Iterable + +import numpy as np +import rasterio +from rasterio.windows import Window + +from satchip import models + + +def make_grid_from_reference(reference: Path, chip_size = 256) -> list[models.GridCell]: + grid = [] + + with rasterio.open(reference) as ref: + n_cols = ref.width // chip_size + n_rows = ref.height // chip_size + + for row in range(n_rows): + for col in range(n_cols): + window = Window(col * chip_size, row * chip_size, chip_size, chip_size) + bounds = ref.window_bounds(window) + + cell_id = f"{row:03d}.{col:03d}" + + grid.append(models.GridCell(cell_id, bounds)) + + return grid + + +def chip_data(grid: list[models.GridCell], layers: Iterable[Path], output_path: Path): + chips = {} + + for layer in layers: + with rasterio.open(layer) as src: + for tile_id, bounds in grid: + window = src.window(*bounds) + window = Window( + round(window.col_off), + round(window.row_off), + round(window.width), + round(window.height), + ) + + data = src.read(window=window) + + chip_meta = src.meta.copy() + chip_meta.update( + { + "width": window.width, + "height": window.height, + "transform": src.window_transform(window), + } + ) + + chip_name = f"{tile_id}.{layer.name}" + chip_path = output_path / chip_name + + with rasterio.open(chip_path, "w", **chip_meta) as dst: + dst.write(data) + + chips[tile_id] = chip_path + + return chips + + +def filter_chips(chips, modality): + if modality.id == 'HLS': + filtered_chips = filter_hls_chips(chips) + elif modality.id == 'RTC': + filtered_chips = filter_rtc_chips(chips) + + return filtered_chips + + +def filter_hls_chips(chips: dict[str, dict]) -> list[dict]: + good_chips = [] + + for tile_id, chip in chips.items(): + with rasterio.open(chip["Fmask"]) as ds: + qc = clear_px_Fmask(ds.read(1)) + + with rasterio.open(chip["EVENT"]) as ds: + event = ds.read(1) + + ny, nx = qc.shape + n_px = 1.0 * ny * nx + + # cloud-free pixels (0 clear, 1 cloud, 255 nodata) + n_cf = len(np.where(qc == 0)[0]) + pct_cf = 100.0 * (n_cf / n_px) + + # event pixels + n_ev = len(np.where(event > 0)[0]) + + # pct of chip in event + pct_ev = 100.0 * (n_ev / n_px) if n_ev > 0 else 0 + + if pct_cf > 95 and pct_ev > 1: + good_chips.append(chip) + + return good_chips + + +def bytescale(arr, cmin=0, cmax=1, low=0, high=255): + # clip the data to be in the range of cmin to cmax + arr = np.clip(arr, cmin, cmax) + high = float(high) + low = float(low) + cmax = float(cmax) + cmin = float(cmin) + m = (high - low) / (cmax - cmin) # slope + b = high - (m * cmax) # intercept + arr = np.uint8((m * arr) + b) + return arr + + +def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: + fmask_clear = np.array([ + 0, 4, 16, 20, 32, 36, 48, 52, + 64, 68, 80, 84, 96, 100, 112, 116, + 128, 132, 144, 148, 160, 164, 176, 180, + 192, 196, 208, 212, 224, 228, 240, 244 + ], dtype=Fmask.dtype) + + cloudmask = np.ones_like(Fmask, dtype=np.uint8) + cloudmask[np.isin(Fmask, fmask_clear)] = 0 + cloudmask[Fmask == 255] = 255 + + return cloudmask + + +def filter_rtc_chips(chips: dict[str, dict]) -> list[dict]: + good_chips = [] + + for tile_id, chip in chips.items(): + with rasterio.open(chip["BANDS"]) as ds: + rtc_data = ds.read() + + with rasterio.open(chip["EVENT"]) as ds: + event_mask = ds.read(1) + + has_nan_pixels = np.isnan(rtc_data).sum() > 0 + + num_pixels = event_mask.size + num_event_pixels = np.count_nonzero(event_mask > 0) + + pct_pixels_over_event = 100.0 * (num_event_pixels / num_pixels) + data_overlaps_event = pct_pixels_over_event > 1 + + if not has_nan_pixels and data_overlaps_event: + good_chips.append(chip) + + return good_chips diff --git a/src/satchip/models.py b/src/satchip/models.py index f75049c..9e70919 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -1,8 +1,10 @@ +from pathlib import Path from datetime import datetime from dataclasses import dataclass from typing import TypedDict, NamedTuple from typing import Tuple +from rasterio.coords import BoundingBox import shapely import pyproj @@ -38,6 +40,27 @@ def buffered_geometry(self) -> shapely.geometry.Polygon: return shapely.ops.transform(to_wgs84.transform, buffered) +@dataclass(frozen=True) +class GridCell: + id: str + bounds: BoundingBox + + +@dataclass(frozen=True) +class Layer: + name: str + modality: Modality + path: Path + + +@dataclass(frozen=True) +class Chip: + id: str + modality: Modality + path: Path + + + class ModalityError(Exception): pass diff --git a/tests/conftest.py b/tests/conftest.py index e521051..1ed5ba7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,17 @@ def pristine_gdf(tmp_path): return df +@pytest.fixture(scope="session") +def warped_event_files(): + warped_base = DATA_PATH / "warped" + + return [ + warped_base / '102a.HLS_S30.2019-07-09.Fmask.tif', + warped_base / '102a.MASK.tif', + warped_base / '102a.stacked.tif', + ] + + @pytest.fixture(scope="session") def s2_local_files(s2_event): download_path = DATA_PATH / "raw" diff --git a/tests/test_chip_data.py b/tests/test_chip_data.py new file mode 100644 index 0000000..e10f7b8 --- /dev/null +++ b/tests/test_chip_data.py @@ -0,0 +1,19 @@ +from satchip import chip_data + + +def test_make_grid(warped_event_files): + label = [f for f in warped_event_files if 'MASK' in f.name].pop() + + grid = chip_data.make_grid_from_reference(label) + assert len(grid) == 27 + + grid = chip_data.make_grid_from_reference(label, chip_size=512) + assert len(grid) == 4 + + grids = (tuple(chip_data.make_grid_from_reference(f)) for f in warped_event_files) + assert len(set(grids)) == 1 + + +def test_chip_data(warped_event_files): + label = [f for f in warped_event_files if 'MASK' in f.name].pop() + grid = chip_data.make_grid_from_reference(label) From 2439b1bf7b391053a806a846fe1069c44e54e209 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 3 Apr 2026 14:12:53 -0800 Subject: [PATCH 15/21] Add chipping function with tests --- src/satchip/chip_data.py | 67 +++++++++++++++++++++------------------- src/satchip/models.py | 2 -- tests/test_chip_data.py | 20 +++++++++++- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/src/satchip/chip_data.py b/src/satchip/chip_data.py index 58e4d5e..c15f953 100644 --- a/src/satchip/chip_data.py +++ b/src/satchip/chip_data.py @@ -27,38 +27,41 @@ def make_grid_from_reference(reference: Path, chip_size = 256) -> list[models.Gr return grid -def chip_data(grid: list[models.GridCell], layers: Iterable[Path], output_path: Path): - chips = {} - - for layer in layers: - with rasterio.open(layer) as src: - for tile_id, bounds in grid: - window = src.window(*bounds) - window = Window( - round(window.col_off), - round(window.row_off), - round(window.width), - round(window.height), - ) - - data = src.read(window=window) - - chip_meta = src.meta.copy() - chip_meta.update( - { - "width": window.width, - "height": window.height, - "transform": src.window_transform(window), - } - ) - - chip_name = f"{tile_id}.{layer.name}" - chip_path = output_path / chip_name - - with rasterio.open(chip_path, "w", **chip_meta) as dst: - dst.write(data) - - chips[tile_id] = chip_path +def chip_data(grid: list[models.GridCell], layer: Path, output_path: Path) -> list[models.Chips]: + output_path.mkdir(exist_ok=True, parents=True) + + chips = [] + + with rasterio.open(layer) as src: + for grid_cell in grid: + window = src.window(*grid_cell.bounds) + window = Window( + round(window.col_off), + round(window.row_off), + round(window.width), + round(window.height), + ) + + data = src.read(window=window) + + chip_meta = src.meta.copy() + chip_meta.update( + { + "width": window.width, + "height": window.height, + "transform": src.window_transform(window), + } + ) + + chip_name = f"{grid_cell.id}.{layer.name}" + chip_path = output_path / chip_name + + with rasterio.open(chip_path, "w", **chip_meta) as dst: + dst.write(data) + + chip = models.Chip(grid_cell.id, chip_path) + + chips.append(chip) return chips diff --git a/src/satchip/models.py b/src/satchip/models.py index 9e70919..43f0cd3 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -56,11 +56,9 @@ class Layer: @dataclass(frozen=True) class Chip: id: str - modality: Modality path: Path - class ModalityError(Exception): pass diff --git a/tests/test_chip_data.py b/tests/test_chip_data.py index e10f7b8..99fda3c 100644 --- a/tests/test_chip_data.py +++ b/tests/test_chip_data.py @@ -1,3 +1,5 @@ +import rasterio + from satchip import chip_data @@ -14,6 +16,22 @@ def test_make_grid(warped_event_files): assert len(set(grids)) == 1 -def test_chip_data(warped_event_files): +def test_chip_data(warped_event_files, tmp_path): label = [f for f in warped_event_files if 'MASK' in f.name].pop() grid = chip_data.make_grid_from_reference(label) + + for file in warped_event_files: + chips = chip_data.chip_data(grid, file, tmp_path / 'chips') + + chip = chips[0] + assert file.name in chip.path.name + assert chip.id in chip.path.name + + assert len(chips) == len(grid) + + shapes = set() + for chip in chips: + with rasterio.open(chip.path) as ds: + shapes.add(ds.shape) + + assert shapes == {(256, 256)} From ad0991d8b8421cd9dd5e538bc991800cc64b258e Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 3 Apr 2026 17:26:09 -0800 Subject: [PATCH 16/21] Add functionaly to filter chips --- src/satchip/chip_data.py | 98 ++++++++++++++++++++--------------- src/satchip/merge_modality.py | 21 +++++++- src/satchip/models.py | 9 ++++ tests/test_chip_data.py | 37 ++++++++++++- 4 files changed, 120 insertions(+), 45 deletions(-) diff --git a/src/satchip/chip_data.py b/src/satchip/chip_data.py index c15f953..4c60094 100644 --- a/src/satchip/chip_data.py +++ b/src/satchip/chip_data.py @@ -60,48 +60,68 @@ def chip_data(grid: list[models.GridCell], layer: Path, output_path: Path) -> li dst.write(data) chip = models.Chip(grid_cell.id, chip_path) - chips.append(chip) return chips -def filter_chips(chips, modality): - if modality.id == 'HLS': - filtered_chips = filter_hls_chips(chips) - elif modality.id == 'RTC': - filtered_chips = filter_rtc_chips(chips) +def make_chip_stacks(data_chips: list[models.Chip], validation_mask_chips: list[models.Chip], label_chips: list[models.Chip], modality: models.Modality) -> list[models.ChipStack]: + chip_stacks = [] + + for data, mask, label in zip( + sorted(data_chips, key=lambda c: c.id), + sorted(validation_mask_chips, key=lambda c: c.id), + sorted(label_chips, key=lambda c: c.id), + ): + chip_stack = models.ChipStack( + id=data.id, + data=data.path, + validation_mask=mask.path, + label=label.path, + modality=modality, + ) + + chip_stacks.append(chip_stack) - return filtered_chips + return chip_stacks -def filter_hls_chips(chips: dict[str, dict]) -> list[dict]: +def filter_chips(chip_stacks: list[models.ChipStack]) -> list[models.ChipStack]: good_chips = [] - for tile_id, chip in chips.items(): - with rasterio.open(chip["Fmask"]) as ds: - qc = clear_px_Fmask(ds.read(1)) + for chip_stack in chip_stacks: + if 'HLS' in chip_stack.modality['id']: + is_good_chip = is_good_hls_chip(chip_stack) + elif 'RTC' in chip_stack.modality['id']: + is_good_chip = is_good_rtc_chip(chip_stack) - with rasterio.open(chip["EVENT"]) as ds: - event = ds.read(1) + if is_good_chip: + good_chips.append(chip_stack) - ny, nx = qc.shape - n_px = 1.0 * ny * nx + return good_chips - # cloud-free pixels (0 clear, 1 cloud, 255 nodata) - n_cf = len(np.where(qc == 0)[0]) - pct_cf = 100.0 * (n_cf / n_px) - # event pixels - n_ev = len(np.where(event > 0)[0]) +def is_good_hls_chip(chip_stack: models.ChipStack) -> bool: + with rasterio.open(chip_stack.validation_mask) as ds: + qc = clear_px_Fmask(ds.read(1)) - # pct of chip in event - pct_ev = 100.0 * (n_ev / n_px) if n_ev > 0 else 0 + with rasterio.open(chip_stack.label) as ds: + event = ds.read(1) - if pct_cf > 95 and pct_ev > 1: - good_chips.append(chip) + ny, nx = qc.shape + n_px = 1.0 * ny * nx - return good_chips + # cloud-free pixels (0 clear, 1 cloud, 255 nodata) + n_cf = len(np.where(qc == 0)[0]) + pct_cf = 100.0 * (n_cf / n_px) + + # event pixels + n_ev = len(np.where(event > 0)[0]) + + # pct of chip in event + pct_ev = 100.0 * (n_ev / n_px) if n_ev > 0 else 0 + + return pct_cf > 95 and pct_ev > 1 def bytescale(arr, cmin=0, cmax=1, low=0, high=255): @@ -132,25 +152,19 @@ def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: return cloudmask -def filter_rtc_chips(chips: dict[str, dict]) -> list[dict]: - good_chips = [] - - for tile_id, chip in chips.items(): - with rasterio.open(chip["BANDS"]) as ds: - rtc_data = ds.read() - - with rasterio.open(chip["EVENT"]) as ds: - event_mask = ds.read(1) +def is_good_rtc_chip(chip_stack: models.ChipStack) -> bool: + with rasterio.open(chip_stack.data) as ds: + rtc_data = ds.read() - has_nan_pixels = np.isnan(rtc_data).sum() > 0 + with rasterio.open(chip_stack.label) as ds: + event_mask = ds.read(1) - num_pixels = event_mask.size - num_event_pixels = np.count_nonzero(event_mask > 0) + has_nan_pixels = np.isnan(rtc_data).sum() > 0 - pct_pixels_over_event = 100.0 * (num_event_pixels / num_pixels) - data_overlaps_event = pct_pixels_over_event > 1 + num_pixels = event_mask.size + num_event_pixels = np.count_nonzero(event_mask > 0) - if not has_nan_pixels and data_overlaps_event: - good_chips.append(chip) + pct_pixels_over_event = 100.0 * (num_event_pixels / num_pixels) + data_overlaps_event = pct_pixels_over_event > 1 - return good_chips + return not has_nan_pixels and data_overlaps_event diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index c030544..71ab5be 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -145,7 +145,12 @@ def reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: ) -def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_dir: Path, bounding_box_wgs84: tuple(float, float, float, float)) -> list[Path]: +def warp_to_reference( + reference_path: Path, + data_files: Iterable[Path], + output_dir: Path, + bounding_box_wgs84: tuple(float, float, float, float) +) -> list[Path]: output_dir.mkdir(parents=True, exist_ok=True) dst_transform, width, height, dst_crs = _build_common_grid( @@ -168,7 +173,9 @@ def warp_to_reference(reference_path: Path, data_files: Iterable[Path], output_d return output -def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, resampling=Resampling.bilinear) -> Path: +def _warp_single(input_path: Path, output_path: Path, dst_transform: Affine, width: int, height: int, dst_crs: CRS) -> Path: + resampling = _get_resampling_method(input_path) + with rasterio.open(input_path) as src: dst_data = np.zeros((src.count, height, width), dtype=src.dtypes[0]) @@ -198,6 +205,16 @@ def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, return output_path +def _get_resampling_method(filepath: Path) -> Resampling: + filename = filepath.name.lower() + + if "fmask" in filename or "mask" in filename: + return Resampling.nearest + else: + return Resampling.bilinear + + + def _build_common_grid(bounding_box_4326: tuple(float, float, float, float), reference_path: Path) -> tuple[Affine, int, int, CRS]: dst_crs = CRS.from_epsg(4326) minx, miny, maxx, maxy = bounding_box_4326 diff --git a/src/satchip/models.py b/src/satchip/models.py index 43f0cd3..77d2fe5 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -59,6 +59,15 @@ class Chip: path: Path +@dataclass(frozen=True) +class ChipStack: + id: str + data: Path + validation_mask: Path + label: Path + modality: Modality + + class ModalityError(Exception): pass diff --git a/tests/test_chip_data.py b/tests/test_chip_data.py index 99fda3c..a5feed4 100644 --- a/tests/test_chip_data.py +++ b/tests/test_chip_data.py @@ -1,6 +1,6 @@ import rasterio -from satchip import chip_data +from satchip import chip_data, models def test_make_grid(warped_event_files): @@ -35,3 +35,38 @@ def test_chip_data(warped_event_files, tmp_path): shapes.add(ds.shape) assert shapes == {(256, 256)} + + +def test_make_chip_stacks(warped_event_files, tmp_path): + fmask, label, data = warped_event_files + grid = chip_data.make_grid_from_reference(fmask) + + fmask_chips = chip_data.chip_data(grid, fmask, tmp_path / 'chips') + label_chips = chip_data.chip_data(grid, label, tmp_path / 'chips') + data_chips = chip_data.chip_data(grid, data, tmp_path / 'chips') + + stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30) + + assert len(stacks) == len(fmask_chips) + + for stack in stacks: + assert stack.id in stack.validation_mask.name + assert stack.id in stack.data.name + assert stack.id in stack.label.name + assert stack.modality == models.HLS_S30 + + +def test_filter_chips(warped_event_files, tmp_path): + fmask, label, data = warped_event_files + grid = chip_data.make_grid_from_reference(fmask) + + fmask_chips = chip_data.chip_data(grid, fmask, tmp_path / 'chips') + label_chips = chip_data.chip_data(grid, label, tmp_path / 'chips') + data_chips = chip_data.chip_data(grid, data, tmp_path / 'chips') + + stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30) + + filtered = chip_data.filter_chips(stacks) + + assert len(filtered) > 0 + assert len(filtered) < len(stacks) From 0dd2d650b7b3cd12aebf82bc5a6c06b2efaf2c53 Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 6 Apr 2026 09:34:01 -0800 Subject: [PATCH 17/21] Ruff --- notebooks/hwds-example.ipynb | 138 ++++++++++++-------- src/satchip/chip_data.py | 78 +++++++---- src/satchip/download_data.py | 6 +- src/satchip/generate_chips.py | 201 +++++++++++++---------------- src/satchip/generate_labels.py | 12 +- src/satchip/hls.py | 101 ++++++++++----- src/satchip/merge_modality.py | 93 +++++++------ src/satchip/modality.py | 7 +- src/satchip/models.py | 79 ++++++------ src/satchip/mosaic.py | 122 ++++++++--------- src/satchip/old/chip_data.py | 8 +- src/satchip/old/chip_hls.py | 4 +- src/satchip/old/chip_hyp3s1rtc.py | 4 +- src/satchip/old/chip_label.py | 4 +- src/satchip/old/chip_operas1rtc.py | 4 +- src/satchip/old/chip_sentinel2.py | 4 +- src/satchip/old/chip_xr_base.py | 2 +- src/satchip/old/terra_mind_grid.py | 2 +- src/satchip/old/utils.py | 12 +- src/satchip/opera_rtc.py | 22 ++-- tests/conftest.py | 35 ++--- tests/test_download_data.py | 3 +- tests/test_merge_modality.py | 7 +- tests/test_models.py | 48 ++++--- 24 files changed, 521 insertions(+), 475 deletions(-) diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index e9fe7f0..6c7b248 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -21,7 +21,7 @@ "import geopandas as gpd\n", "import pandas as pd\n", "\n", - "from satchip import models, download_data, merge_modality, generate_labels" + "from satchip import models, download_data, merge_modality, generate_labels, chip_data" ] }, { @@ -33,14 +33,15 @@ "source": [ "modality = models.HLS_S30\n", "\n", - "DATA_PATH = Path.cwd() / 'data' \n", + "DATA_PATH = Path.cwd() / 'data'\n", "MODALITY_PATH = DATA_PATH / modality['id']\n", "\n", "RAW_DATA_PATH = MODALITY_PATH / 'raw'\n", "MERGED_PATH = MODALITY_PATH / 'merged'\n", "REPROJECTED_PATH = MODALITY_PATH / 'wgs84'\n", "STACKED_PATH = MODALITY_PATH / 'stacked'\n", - "WARPED_PATH = MODALITY_PATH / 'warped' " + "WARPED_PATH = MODALITY_PATH / 'warped'\n", + "CHIPS_PATH = MODALITY_PATH / 'chips'" ] }, { @@ -50,11 +51,11 @@ "metadata": {}, "outputs": [], "source": [ - "shp_path = \"hwds_pristine\"\n", + "shp_path = 'hwds_pristine'\n", "df = gpd.read_file(shp_path)\n", "\n", - "df[\"SwathDate\"] = pd.to_datetime(df[\"SwathDate\"], format=\"%Y-%m-%d\")\n", - "df[\"HLSDate\"] = pd.to_datetime(df[\"HLSDate\"], format=\"%Y-%m-%d\")" + "df['SwathDate'] = pd.to_datetime(df['SwathDate'], format='%Y-%m-%d')\n", + "df['HLSDate'] = pd.to_datetime(df['HLSDate'], format='%Y-%m-%d')" ] }, { @@ -87,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "6a611425", "metadata": {}, "outputs": [ @@ -95,17 +96,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Event(name='102a', date=Timestamp('2019-07-09 00:00:00'), wgs84_geometry=)\n" + "Event(name='102a', date=Timestamp('2019-07-09 00:00:00'), wgs84_geometry=, buffer_m=10000)\n" ] } ], "source": [ - "event = models.Event(\n", - " name=swath['HLSID'],\n", - " date=swath['SwathDate'],\n", - " wgs84_geometry=swath['geometry'],\n", - " buffer_m=10000\n", - ")\n", + "event = models.Event(name=swath['HLSID'], date=swath['SwathDate'], wgs84_geometry=swath['geometry'], buffer_m=10000)\n", "print(event)" ] }, @@ -126,9 +122,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10439.36it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 191617.95it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 424143.10it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10394.81it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 230879.12it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 373749.86it/s]\n" ] } ], @@ -181,7 +177,7 @@ "outputs": [], "source": [ "stacked_filename = MERGED_PATH / f'{swath[\"HLSID\"]}.stacked.tif'\n", - "data_bands, fmask = merged_event[:-1], merged_event[-1] \n", + "data_bands, fmask = merged_event[:-1], merged_event[-1]\n", "\n", "stacked = merge_modality.stack_bands(data_bands, stacked_filename)" ] @@ -189,32 +185,6 @@ { "cell_type": "code", "execution_count": 10, - "id": "b4099849", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.B.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.G.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.R.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.N.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW1.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW2.tif')]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "data_bands" - ] - }, - { - "cell_type": "code", - "execution_count": 11, "id": "e6f7e30b", "metadata": {}, "outputs": [], @@ -224,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "id": "9d9b6664", "metadata": {}, "outputs": [ @@ -242,18 +212,84 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "3180399a", "metadata": {}, "outputs": [], "source": [ - "warped = merge_modality.warp_to_reference(\n", - " reference_path=label, \n", - " data_files=[stacked, fmask], \n", - " output_dir=WARPED_PATH, \n", - " bounding_box_4326=event.wgs84_geometry.bounds\n", + "mask, bands, fmask = merge_modality.warp_to_reference(\n", + " reference_path=label,\n", + " data_files=[stacked, fmask],\n", + " output_dir=WARPED_PATH,\n", + " bounding_box_wgs84=event.buffered_geometry().bounds,\n", ")" ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1f65b3ed", + "metadata": {}, + "outputs": [], + "source": [ + "grid = chip_data.make_grid_from_reference(mask)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "3c4cee14", + "metadata": {}, + "outputs": [], + "source": [ + "label_chips = chip_data.chip_data(grid, mask, CHIPS_PATH / 'LABELS')\n", + "data_chips = chip_data.chip_data(grid, bands, CHIPS_PATH / modality['id'])\n", + "fmask_chips = chip_data.chip_data(grid, fmask, CHIPS_PATH / 'FMASK')" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "8d3ac758", + "metadata": {}, + "outputs": [], + "source": [ + "stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30)\n", + "filtered = chip_data.filter_chips(stacks)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "e5c082c9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[ChipStack(id='001.001', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.001.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.001.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.001.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", + " ChipStack(id='001.002', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.002.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.002.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.002.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", + " ChipStack(id='001.003', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.003.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.003.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.003.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", + " ChipStack(id='001.004', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.004.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.004.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.004.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", + " ChipStack(id='001.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.005.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.005.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))})]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "filtered" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b5369e9", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/satchip/chip_data.py b/src/satchip/chip_data.py index 4c60094..03ce27b 100644 --- a/src/satchip/chip_data.py +++ b/src/satchip/chip_data.py @@ -1,5 +1,4 @@ from pathlib import Path -from collections.abc import Iterable import numpy as np import rasterio @@ -8,7 +7,7 @@ from satchip import models -def make_grid_from_reference(reference: Path, chip_size = 256) -> list[models.GridCell]: +def make_grid_from_reference(reference: Path, chip_size: int = 256) -> list[models.GridCell]: grid = [] with rasterio.open(reference) as ref: @@ -20,7 +19,7 @@ def make_grid_from_reference(reference: Path, chip_size = 256) -> list[models.Gr window = Window(col * chip_size, row * chip_size, chip_size, chip_size) bounds = ref.window_bounds(window) - cell_id = f"{row:03d}.{col:03d}" + cell_id = f'{row:03d}.{col:03d}' grid.append(models.GridCell(cell_id, bounds)) @@ -47,16 +46,16 @@ def chip_data(grid: list[models.GridCell], layer: Path, output_path: Path) -> li chip_meta = src.meta.copy() chip_meta.update( { - "width": window.width, - "height": window.height, - "transform": src.window_transform(window), + 'width': window.width, + 'height': window.height, + 'transform': src.window_transform(window), } ) - chip_name = f"{grid_cell.id}.{layer.name}" + chip_name = f'{grid_cell.id}.{layer.name}' chip_path = output_path / chip_name - with rasterio.open(chip_path, "w", **chip_meta) as dst: + with rasterio.open(chip_path, 'w', **chip_meta) as dst: dst.write(data) chip = models.Chip(grid_cell.id, chip_path) @@ -65,7 +64,12 @@ def chip_data(grid: list[models.GridCell], layer: Path, output_path: Path) -> li return chips -def make_chip_stacks(data_chips: list[models.Chip], validation_mask_chips: list[models.Chip], label_chips: list[models.Chip], modality: models.Modality) -> list[models.ChipStack]: +def make_chip_stacks( + data_chips: list[models.Chip], + validation_mask_chips: list[models.Chip], + label_chips: list[models.Chip], + modality: models.Modality, +) -> list[models.ChipStack]: chip_stacks = [] for data, mask, label in zip( @@ -124,26 +128,44 @@ def is_good_hls_chip(chip_stack: models.ChipStack) -> bool: return pct_cf > 95 and pct_ev > 1 -def bytescale(arr, cmin=0, cmax=1, low=0, high=255): - # clip the data to be in the range of cmin to cmax - arr = np.clip(arr, cmin, cmax) - high = float(high) - low = float(low) - cmax = float(cmax) - cmin = float(cmin) - m = (high - low) / (cmax - cmin) # slope - b = high - (m * cmax) # intercept - arr = np.uint8((m * arr) + b) - return arr - - def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: - fmask_clear = np.array([ - 0, 4, 16, 20, 32, 36, 48, 52, - 64, 68, 80, 84, 96, 100, 112, 116, - 128, 132, 144, 148, 160, 164, 176, 180, - 192, 196, 208, 212, 224, 228, 240, 244 - ], dtype=Fmask.dtype) + fmask_clear = np.array( + [ + 0, + 4, + 16, + 20, + 32, + 36, + 48, + 52, + 64, + 68, + 80, + 84, + 96, + 100, + 112, + 116, + 128, + 132, + 144, + 148, + 160, + 164, + 176, + 180, + 192, + 196, + 208, + 212, + 224, + 228, + 240, + 244, + ], + dtype=Fmask.dtype, + ) cloudmask = np.ones_like(Fmask, dtype=np.uint8) cloudmask[np.isin(Fmask, fmask_clear)] = 0 diff --git a/src/satchip/download_data.py b/src/satchip/download_data.py index 906f7c3..73930d8 100644 --- a/src/satchip/download_data.py +++ b/src/satchip/download_data.py @@ -16,9 +16,7 @@ def download_data(event: models.Event, modality: models.Modality, download_path: if not results: return [] - local_files = earthaccess.download( - results, local_path=download_path, show_progress=True - ) + local_files = earthaccess.download(results, local_path=download_path, show_progress=True) return local_files @@ -31,7 +29,7 @@ def _search_data(event: models.Event, modality: models.Modality) -> list[earthac results = earthaccess.search_data( short_name=[collection_id], - temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + temporal=(start_date.strftime('%Y-%m-%d'), final_date.strftime('%Y-%m-%d')), bounding_box=event.buffered_geometry().bounds, ) diff --git a/src/satchip/generate_chips.py b/src/satchip/generate_chips.py index b53a00f..f432766 100644 --- a/src/satchip/generate_chips.py +++ b/src/satchip/generate_chips.py @@ -1,52 +1,51 @@ -from pathlib import Path import shutil import zipfile - +from pathlib import Path import cartopy.crs as ccrs import earthaccess import gdown import geopandas as gpd +import hls import matplotlib.pyplot as plt +import mosaic import numpy as np +import opera_rtc import pandas as pd import rasterio +from modality import Modality from rasterio import features from rasterio.windows import Window from shapely.geometry import box from sklearn.model_selection import train_test_split -from modality import Modality -import mosaic -import hls -import opera_rtc QUITE = True CHIP_SIZE = 256 RNG_SEED = 42 -MODALITY = "RTC" -ALL_BANDS = ("VV", "VH", "mask") -STACK_BANDS = ("VV", "VH") -CHIP_BANDS = ("BANDS", "EVENT", "MASK") +MODALITY = 'RTC' +ALL_BANDS = ('VV', 'VH', 'mask') +STACK_BANDS = ('VV', 'VH') +CHIP_BANDS = ('BANDS', 'EVENT', 'MASK') -MODALITY = "HLS" -ALL_BANDS = ("B", "G", "R", "N", "SW1", "SW2", "Fmask") -STACK_BANDS = ("B", "G", "R", "N", "SW1", "SW2") -CHIP_BANDS = ("BANDS", "EVENT", "MASK", "Fmask") +MODALITY = 'HLS' +ALL_BANDS = ('B', 'G', 'R', 'N', 'SW1', 'SW2', 'Fmask') +STACK_BANDS = ('B', 'G', 'R', 'N', 'SW1', 'SW2') +CHIP_BANDS = ('BANDS', 'EVENT', 'MASK', 'Fmask') SHOULD_CLEANUP = False def main(modalities: list[Modality]): - hwds_path = Path("hwds") + hwds_path = Path('hwds') print('Making folders') data_paths = { - "CHIPS_ALL": hwds_path / "CHIPS_ALL", - "CHIPS": hwds_path / "CHIPS", - "MERGED": hwds_path / "MERGED", - "PLOTS": hwds_path / 'PLOTS' + 'CHIPS_ALL': hwds_path / 'CHIPS_ALL', + 'CHIPS': hwds_path / 'CHIPS', + 'MERGED': hwds_path / 'MERGED', + 'PLOTS': hwds_path / 'PLOTS', } if SHOULD_CLEANUP: @@ -56,7 +55,7 @@ def main(modalities: list[Modality]): item.unlink() - for p in ("CHIPS", "CHIPS_ALL"): + for p in ('CHIPS', 'CHIPS_ALL'): shutil.rmtree(data_paths[p], ignore_errors=True) for p in data_paths.values(): @@ -64,168 +63,159 @@ def main(modalities: list[Modality]): gdf = _load_event_database(hwds_path) gdf_utm = gdf.to_crs(32615) - gdf["buffered_event"] = gdf_utm.buffer(3000).to_crs(4326) - gdf["buffered_event_background"] = gdf_utm.buffer(10000).to_crs(4326) + gdf['buffered_event'] = gdf_utm.buffer(3000).to_crs(4326) + gdf['buffered_event_background'] = gdf_utm.buffer(10000).to_crs(4326) gdf = gdf.to_crs(4326) # keepers = [1442, 622, 1079, 628] keepers = [1442, 622] - gdf = gdf[gdf["swathID"].isin(keepers)] + gdf = gdf[gdf['swathID'].isin(keepers)] earthaccess.login() tm_chips = [] for i, (swathID, swath) in enumerate(gdf.iterrows(), start=1): - swathID = f"{int(swath['swathID']):04d}" + swathID = f'{int(swath["swathID"]):04d}' print(f'Processing Swath {swathID} ({i} / {len(gdf)})') merged = mosaic.data_over_swath(swath, modalities, output_path=data_paths['MERGED']) template_path = merged[modalities[0].id]['BANDS'] - event_tif, mask_tif = _generate_masks( - template_path, swathID, swath - ) + event_tif, mask_tif = _generate_masks(template_path, swathID, swath) merged_data = { **merged, - "EVENT": event_tif, - "MASK": mask_tif, + 'EVENT': event_tif, + 'MASK': mask_tif, } if not all(is_valid_data(merged_data, modality) for modality in modalities): - print("Skipping: not enough valid data") + print('Skipping: not enough valid data') continue for modality in modalities: - print(f"Chipping {modality.id}!") + print(f'Chipping {modality.id}!') chips = _chip_data(merged_data, data_paths['CHIPS_ALL'], modality) good_chips = filter_chips(chips, modality) - print(f"Found {len(good_chips)} good chips") + print(f'Found {len(good_chips)} good chips') for chip in good_chips: for band, chip_path in chip.items(): - if band not in ("MASK", "BANDS"): + if band not in ('MASK', 'BANDS'): continue - dest = data_paths["CHIPS"] / chip_path.name + dest = data_paths['CHIPS'] / chip_path.name shutil.copy(chip_path, dest) tm_chips += good_chips for _, swath in gdf.iterrows(): - swath_id = _make_swath_id(swath["swathID"]) + swath_id = _make_swath_id(swath['swathID']) for modality in modalities: - merged_file = list(data_paths["MERGED"].glob(f"{swath_id}.{modality.id}.*.BANDS.tif")) + merged_file = list(data_paths['MERGED'].glob(f'{swath_id}.{modality.id}.*.BANDS.tif')) if len(merged_file) == 0: - print(f"no chips for {swath_id}") + print(f'no chips for {swath_id}') continue - all_chips = list(data_paths["CHIPS_ALL"].glob(f"*.{swath_id}.{modality.id}.*.tif")) - good_chips = list(data_paths["CHIPS"].glob(f"*.{swath_id}.{modality.id}.*.tif")) + all_chips = list(data_paths['CHIPS_ALL'].glob(f'*.{swath_id}.{modality.id}.*.tif')) + good_chips = list(data_paths['CHIPS'].glob(f'*.{swath_id}.{modality.id}.*.tif')) - print(f"plotting {swath_id}") - _plot_chips( - merged_file[0], all_chips, good_chips, swath, modality, save_to=data_paths["PLOTS"] - ) + print(f'plotting {swath_id}') + _plot_chips(merged_file[0], all_chips, good_chips, swath, modality, save_to=data_paths['PLOTS']) for modality in modalities: print(f'Calulating stats for modality: {modality.id}') - band_chips = list(data_paths["CHIPS"].glob(f"*.{modality.id}.*.BANDS.tif")) + band_chips = list(data_paths['CHIPS'].glob(f'*.{modality.id}.*.BANDS.tif')) means, stds = calculate_stats(chips=band_chips, n_bands=len(modality.stack_bands)) - means_str = ", ".join(f"{x:.4f}" for x in means) - stds_str = ", ".join(f"{x:.4f}" for x in stds) + means_str = ', '.join(f'{x:.4f}' for x in means) + stds_str = ', '.join(f'{x:.4f}' for x in stds) - stats_str = ( - f"Means {modality.stack_bands}: {means_str}\n" - f"Stds {modality.stack_bands}: {stds_str}\n" - ) + stats_str = f'Means {modality.stack_bands}: {means_str}\nStds {modality.stack_bands}: {stds_str}\n' (hwds_path / '{modality.id}-statistics.txt').write_text(stats_str) print(stats_str) -def _generate_masks( - template_data_path: Path, swathID: str, swath: pd.Series -) -> tuple[Path, Path]: - event_path = template_data_path.parent / f"{swathID}.EVENT.tif" - mask_path = template_data_path.parent / f"{swathID}.MASK.tif" +def _generate_masks(template_data_path: Path, swathID: str, swath: pd.Series) -> tuple[Path, Path]: + event_path = template_data_path.parent / f'{swathID}.EVENT.tif' + mask_path = template_data_path.parent / f'{swathID}.MASK.tif' with rasterio.open(template_data_path) as ds: profile = ds.profile mask_raster = features.rasterize( - shapes=[[swath["geometry"], 1]], + shapes=[[swath['geometry'], 1]], fill=0, out_shape=ds.shape, transform=ds.transform, ) - with rasterio.open(mask_path, "w", **profile) as dst: + with rasterio.open(mask_path, 'w', **profile) as dst: dst.write(mask_raster, 1) - print("generated:", mask_path) + print('generated:', mask_path) event_mask = features.rasterize( shapes=[ - [swath["buffered_event_background"], 3], - [swath["buffered_event"], 2], - [swath["geometry"], 1], + [swath['buffered_event_background'], 3], + [swath['buffered_event'], 2], + [swath['geometry'], 1], ], fill=0, out_shape=ds.shape, transform=ds.transform, ) - with rasterio.open(event_path, "w", **profile) as dst: + with rasterio.open(event_path, 'w', **profile) as dst: dst.write(event_mask, 1) - print("generated:", event_path) + print('generated:', event_path) return event_path, mask_path def _load_event_database(data_dir: Path): # use 60-swath version - hwds_google_drive_id = "1h_JIEcrrUF3OSTrmwAKNPa0eUEhPA2Xx" - drive_url = f"https://drive.google.com/uc?id={hwds_google_drive_id}" + hwds_google_drive_id = '1h_JIEcrrUF3OSTrmwAKNPa0eUEhPA2Xx' + drive_url = f'https://drive.google.com/uc?id={hwds_google_drive_id}' - shp_dir = data_dir / "SHP" + shp_dir = data_dir / 'SHP' shp_dir.mkdir(parents=True, exist_ok=True) - filename = "hwds_v3_20250205_subset_60.zip" + filename = 'hwds_v3_20250205_subset_60.zip' zip_path = shp_dir / filename if not zip_path.exists(): gdown.download(drive_url, str(zip_path), quiet=False) - with zipfile.ZipFile(zip_path, "r") as zip_ref: + with zipfile.ZipFile(zip_path, 'r') as zip_ref: zip_ref.extractall(path=shp_dir) - shp_path = shp_dir / "hwds_v3_20250205_subset_60.shp" + shp_path = shp_dir / 'hwds_v3_20250205_subset_60.shp' gdf = gpd.read_file(shp_path) - gdf["swathDate"] = pd.to_datetime(gdf["swathDate"], format="%Y-%m-%d") - gdf["ls5hlsDate"] = pd.to_datetime(gdf["ls5hlsDate"], format="%Y-%m-%d") - gdf["s1Date"] = pd.to_datetime(gdf["s1Date"], format="%Y-%m-%d") + gdf['swathDate'] = pd.to_datetime(gdf['swathDate'], format='%Y-%m-%d') + gdf['ls5hlsDate'] = pd.to_datetime(gdf['ls5hlsDate'], format='%Y-%m-%d') + gdf['s1Date'] = pd.to_datetime(gdf['s1Date'], format='%Y-%m-%d') return gdf def create_split_files(band_chips: list[Path], splits_path: Path) -> None: - chip_ids = [p.name.removesuffix("BANDS.tif") for p in band_chips] + chip_ids = [p.name.removesuffix('BANDS.tif') for p in band_chips] the_rest, test = train_test_split(chip_ids, test_size=0.15, random_state=RNG_SEED) train, val = train_test_split(the_rest, test_size=0.15, random_state=RNG_SEED) - splits = {"train": train, "val": val, "test": test} + splits = {'train': train, 'val': val, 'test': test} for split, chip_ids in splits.items(): - split_path = splits_path / f"{split}.txt" - split_path.write_text("\n".join(chip_ids)) + split_path = splits_path / f'{split}.txt' + split_path.write_text('\n'.join(chip_ids)) def calculate_stats(chips: list[Path], n_bands: int = 2) -> tuple: @@ -248,11 +238,7 @@ def calculate_stats(chips: list[Path], n_bands: int = 2) -> tuple: total_count = count + batch_count mean = mean + delta * (batch_count / total_count) - M2 = ( - M2 - + batch_var * batch_count - + (delta**2) * count * batch_count / total_count - ) + M2 = M2 + batch_var * batch_count + (delta**2) * count * batch_count / total_count count = total_count variance = M2 / count @@ -283,17 +269,15 @@ def _plot_chips( fig, ax = plt.subplots( 1, 1, - subplot_kw={"projection": crs_pc}, + subplot_kw={'projection': crs_pc}, figsize=(12, 12), - layout="constrained", + layout='constrained', ) - swath_geom = swath["geometry"] + swath_geom = swath['geometry'] - ax.imshow(img, extent=full_extent, origin="upper", transform=crs_pc) - ax.add_geometries( - [swath_geom], edgecolor="red", linewidth=2, facecolor="none", crs=crs_pc - ) + ax.imshow(img, extent=full_extent, origin='upper', transform=crs_pc) + ax.add_geometries([swath_geom], edgecolor='red', linewidth=2, facecolor='none', crs=crs_pc) def show_chips(chips, color, linewidth, z): for chip in chips: @@ -312,20 +296,20 @@ def show_chips(chips, color, linewidth, z): linewidth=linewidth, alpha=1, zorder=z, - facecolor="none", + facecolor='none', crs=crs_pc, ) - show_chips(all_chips, "yellow", 1, z=1) - show_chips(good_chips, "blue", 3, z=2) + show_chips(all_chips, 'yellow', 1, z=1) + show_chips(good_chips, 'blue', 3, z=2) ax.set_extent(full_extent, crs=crs_pc) if save_to: plt.savefig( - save_to / f"{merged_band_file.name.removesuffix('BANDS.tif')}.png", + save_to / f'{merged_band_file.name.removesuffix("BANDS.tif")}.png', dpi=300, - bbox_inches="tight", + bbox_inches='tight', ) if not quite: @@ -335,14 +319,14 @@ def show_chips(chips, color, linewidth, z): def _make_swath_id(swathID): - return f"{int(swathID):04d}" + return f'{int(swathID):04d}' def _chip_data(merged, output_path: Path, modality: Modality, chip_size=CHIP_SIZE): chips = {} grid = [] - with rasterio.open(merged[modality.id]["BANDS"]) as ref: + with rasterio.open(merged[modality.id]['BANDS']) as ref: n_cols = ref.width // chip_size n_rows = ref.height // chip_size @@ -351,7 +335,7 @@ def _chip_data(merged, output_path: Path, modality: Modality, chip_size=CHIP_SIZ window = Window(col * chip_size, row * chip_size, chip_size, chip_size) bounds = ref.window_bounds(window) - tile_id = f"{row:03d}.{col:03d}" + tile_id = f'{row:03d}.{col:03d}' chips[tile_id] = {} grid.append((tile_id, bounds)) @@ -379,16 +363,16 @@ def _chip_data(merged, output_path: Path, modality: Modality, chip_size=CHIP_SIZ chip_meta = src.meta.copy() chip_meta.update( { - "width": window.width, - "height": window.height, - "transform": src.window_transform(window), + 'width': window.width, + 'height': window.height, + 'transform': src.window_transform(window), } ) - chip_name = f"{tile_id}.{layer_path.name}" + chip_name = f'{tile_id}.{layer_path.name}' chip_path = output_path / chip_name - with rasterio.open(chip_path, "w", **chip_meta) as dst: + with rasterio.open(chip_path, 'w', **chip_meta) as dst: dst.write(data) chips[tile_id][chip_layer] = chip_path @@ -432,19 +416,16 @@ def data_transform(data, modality): return data -if __name__ == "__main__": +if __name__ == '__main__': opera_rtc_mod = Modality( - id="RTC", - all_bands=("VV", "VH", "mask"), - stack_bands=("VV", "VH"), - chip_bands=("BANDS", "EVENT", "MASK") + id='RTC', all_bands=('VV', 'VH', 'mask'), stack_bands=('VV', 'VH'), chip_bands=('BANDS', 'EVENT', 'MASK') ) hls_mod = Modality( - id="HLS", - all_bands=("B", "G", "R", "N", "SW1", "SW2", "Fmask"), - stack_bands=("B", "G", "R", "N", "SW1", "SW2"), - chip_bands=("BANDS", "EVENT", "MASK", "Fmask") + id='HLS', + all_bands=('B', 'G', 'R', 'N', 'SW1', 'SW2', 'Fmask'), + stack_bands=('B', 'G', 'R', 'N', 'SW1', 'SW2'), + chip_bands=('BANDS', 'EVENT', 'MASK', 'Fmask'), ) modalities = [hls_mod, opera_rtc_mod] diff --git a/src/satchip/generate_labels.py b/src/satchip/generate_labels.py index 6c5cf55..6c23fe8 100644 --- a/src/satchip/generate_labels.py +++ b/src/satchip/generate_labels.py @@ -6,12 +6,10 @@ from satchip import models -def binary_mask_from_template( - template_data_path: Path, event: models.Event, output_dir: Path -) -> tuple[Path, Path]: +def binary_mask_from_template(template_data_path: Path, event: models.Event, output_dir: Path) -> tuple[Path, Path]: output_dir.mkdir(exist_ok=True, parents=True) - mask_path = output_dir / f"{event.name}.MASK.tif" + mask_path = output_dir / f'{event.name}.MASK.tif' with rasterio.open(template_data_path) as ds: profile = ds.profile @@ -23,10 +21,10 @@ def binary_mask_from_template( transform=ds.transform, ) - profile.update(dtype="uint8", count=1, nodata=255) + profile.update(dtype='uint8', count=1, nodata=255) - with rasterio.open(mask_path, "w", **profile) as dst: + with rasterio.open(mask_path, 'w', **profile) as dst: dst.write(mask_raster, 1) - print("generated:", mask_path) + print('generated:', mask_path) return mask_path diff --git a/src/satchip/hls.py b/src/satchip/hls.py index 1f7022b..7f8478d 100644 --- a/src/satchip/hls.py +++ b/src/satchip/hls.py @@ -1,21 +1,21 @@ -from datetime import timedelta, datetime +from datetime import datetime, timedelta from pathlib import Path import earthaccess -from earthaccess.results import DataGranule import numpy as np import rasterio +from earthaccess.results import DataGranule def search_hls_data(start_date: datetime, bounding_box: tuple[float, float, float, float]) -> list[DataGranule]: final_date = start_date + timedelta(days=1) # collection_ids = ["C2021957295-LPCLOUD"] # S2 - collection_ids = ["C2021957657-LPCLOUD", "C2021957295-LPCLOUD"] # S2, L30 + collection_ids = ['C2021957657-LPCLOUD', 'C2021957295-LPCLOUD'] # S2, L30 results = earthaccess.search_data( concept_id=collection_ids, - temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + temporal=(start_date.strftime('%Y-%m-%d'), final_date.strftime('%Y-%m-%d')), bounding_box=bounding_box, cloud_hosted=True, ) @@ -30,24 +30,24 @@ def band_from_hls_filename(filename): sensor, band = parts[1], parts[-2] bands = { - "L30": { - "B02": "B", - "B03": "G", - "B04": "R", - "B05": "N", - "B06": "SW1", - "B07": "SW2", - "Fmask": "Fmask", + 'L30': { + 'B02': 'B', + 'B03': 'G', + 'B04': 'R', + 'B05': 'N', + 'B06': 'SW1', + 'B07': 'SW2', + 'Fmask': 'Fmask', + }, + 'S30': { + 'B02': 'B', + 'B03': 'G', + 'B04': 'R', + 'B08': 'N', + 'B11': 'SW1', + 'B12': 'SW2', + 'Fmask': 'Fmask', }, - "S30": { - "B02": "B", - "B03": "G", - "B04": "R", - "B08": "N", - "B11": "SW1", - "B12": "SW2", - "Fmask": "Fmask", - } } try: @@ -57,21 +57,52 @@ def band_from_hls_filename(filename): def make_merged_hls_name(template_filename: str) -> str: - parts = template_filename.split(".") + parts = template_filename.split('.') parts[4] = parts[4][0:7] parts.pop(3) parts[-2] = band_from_hls_filename(template_filename) - f_template_merge = ".".join(parts) + f_template_merge = '.'.join(parts) return f_template_merge def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: - fmask_clear = np.array([ - 0, 4, 16, 20, 32, 36, 48, 52, - 64, 68, 80, 84, 96, 100, 112, 116, - 128, 132, 144, 148, 160, 164, 176, 180, - 192, 196, 208, 212, 224, 228, 240, 244 - ], dtype=Fmask.dtype) + fmask_clear = np.array( + [ + 0, + 4, + 16, + 20, + 32, + 36, + 48, + 52, + 64, + 68, + 80, + 84, + 96, + 100, + 112, + 116, + 128, + 132, + 144, + 148, + 160, + 164, + 176, + 180, + 192, + 196, + 208, + 212, + 224, + 228, + 240, + 244, + ], + dtype=Fmask.dtype, + ) cloudmask = np.ones_like(Fmask, dtype=np.uint8) cloudmask[np.isin(Fmask, fmask_clear)] = 0 @@ -92,7 +123,7 @@ def is_valid_hls(fmask_path: Path, event_path: Path): print(qc_profile, event_profile) ny, nx = np.shape(qc) - mask = np.zeros((ny, nx), "uint8") + mask = np.zeros((ny, nx), 'uint8') ok = np.where((event_mask == 2) & (qc == 0)) n_cf_event = len(ok[0]) @@ -101,15 +132,15 @@ def is_valid_hls(fmask_path: Path, event_path: Path): ok = np.where((event_mask == 1) & (qc != 255)) n_valid_event = len(ok[0]) - ok = np.where((event_mask == 1)) + ok = np.where(event_mask == 1) n_event = len(ok[0]) pct_cf_event = 0 if n_valid_event == 0: - print("No coverage") + print('No coverage') else: pct_cf_event = 100.0 * (n_cf_event / n_event) - print("Percent CF/valid in Event:", pct_cf_event) + print('Percent CF/valid in Event:', pct_cf_event) return pct_cf_event > 50 @@ -118,10 +149,10 @@ def filter_hls_chips(chips: dict[str, dict]) -> list[dict]: good_chips = [] for tile_id, chip in chips.items(): - with rasterio.open(chip["Fmask"]) as ds: + with rasterio.open(chip['Fmask']) as ds: qc = clear_px_Fmask(ds.read(1)) - with rasterio.open(chip["EVENT"]) as ds: + with rasterio.open(chip['EVENT']) as ds: event = ds.read(1) ny, nx = qc.shape diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 71ab5be..0c71a81 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -5,10 +5,9 @@ import numpy as np import rasterio from rasterio.crs import CRS -from rasterio.warp import Resampling, transform_bounds -from rasterio.warp import calculate_default_transform, reproject from rasterio.merge import merge -from rasterio.transform import from_bounds, Affine +from rasterio.transform import Affine, from_bounds +from rasterio.warp import Resampling, calculate_default_transform, reproject, transform_bounds from satchip import models @@ -18,12 +17,12 @@ def merge_modality( modality: models.Modality, event: models.Event, output_path: Path, - selected_bands: list[models.Band] | None = None + selected_bands: list[models.Band] | None = None, ) -> list[Path]: output_path.mkdir(exist_ok=True, parents=True) if len(modality_files) == 0: - print(f"Warning: no data for {event.name}") + print(f'Warning: no data for {event.name}') return [] if selected_bands is None: @@ -36,9 +35,7 @@ def merge_modality( merged_name = _make_merge_name(event.name, event.date, band.shortname, modality['id']) - merged_band_path = _merge( - band_files, output_file=output_path / merged_name - ) + merged_band_path = _merge(band_files, output_file=output_path / merged_name) merged.append(merged_band_path) @@ -67,15 +64,15 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: out_meta.update( { - "driver": "GTiff", - "height": mosaic.shape[0], - "width": mosaic.shape[1], - "transform": out_trans, - "crs": band_datasets[0].crs, + 'driver': 'GTiff', + 'height': mosaic.shape[0], + 'width': mosaic.shape[1], + 'transform': out_trans, + 'crs': band_datasets[0].crs, } ) - with rasterio.open(output_file, "w", **out_meta) as dst: + with rasterio.open(output_file, 'w', **out_meta) as dst: dst.write(mosaic, 1) finally: for ds in band_datasets: @@ -92,29 +89,24 @@ def stack_bands(band_files: Iterable[Path], stacked_filename: Path) -> Path: meta.update(count=len(band_files), dtype=np.float32) - with rasterio.open(stacked_filename, "w", **meta) as dst: + with rasterio.open(stacked_filename, 'w', **meta) as dst: for idx, band_file in enumerate(band_files, start=1): with rasterio.open(band_file) as src: - dst.write(src.read(1), idx) return stacked_filename -def reproject_files( - files: list[Path], output_dir: Path -) -> list[Path]: +def reproject_files(files: list[Path], output_dir: Path) -> list[Path]: output_dir.mkdir(exist_ok=True, parents=True) - reprojected_paths = [ - output_dir / f"{file.name}" for file in files - ] + reprojected_paths = [output_dir / f'{file.name}' for file in files] for file, output in zip(files, reprojected_paths): if output.exists(): continue - print(f"reprojecting to wgs84: {output.name}") + print(f'reprojecting to wgs84: {output.name}') reproject_file(file, output) return reprojected_paths @@ -124,16 +116,12 @@ def reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None: # https://rasterio.readthedocs.io/en/stable/topics/reproject.html#reprojecting-a-geotiff-dataset with rasterio.open(local_file) as src: dst_crs = CRS.from_epsg(epsg) - transform, width, height = calculate_default_transform( - src.crs, dst_crs, src.width, src.height, *src.bounds - ) + transform, width, height = calculate_default_transform(src.crs, dst_crs, src.width, src.height, *src.bounds) dst_kwargs = src.meta.copy() - dst_kwargs.update( - {"crs": dst_crs, "transform": transform, "width": width, "height": height} - ) + dst_kwargs.update({'crs': dst_crs, 'transform': transform, 'width': width, 'height': height}) - with rasterio.open(reprojected_file, "w", **dst_kwargs) as dst: + with rasterio.open(reprojected_file, 'w', **dst_kwargs) as dst: for i in range(1, src.count + 1): reproject( source=rasterio.band(src, i), @@ -149,13 +137,11 @@ def warp_to_reference( reference_path: Path, data_files: Iterable[Path], output_dir: Path, - bounding_box_wgs84: tuple(float, float, float, float) + bounding_box_wgs84: tuple(float, float, float, float), ) -> list[Path]: output_dir.mkdir(parents=True, exist_ok=True) - dst_transform, width, height, dst_crs = _build_common_grid( - bounding_box_wgs84, reference_path - ) + dst_transform, width, height, dst_crs = _build_common_grid(bounding_box_wgs84, reference_path) files = (reference_path, *data_files) @@ -164,8 +150,12 @@ def warp_to_reference( out_path = output_dir / data_file.name _warp_single( - data_file, out_path, - dst_transform, width, height, dst_crs, + data_file, + out_path, + dst_transform, + width, + height, + dst_crs, ) output.append(out_path) @@ -173,7 +163,9 @@ def warp_to_reference( return output -def _warp_single(input_path: Path, output_path: Path, dst_transform: Affine, width: int, height: int, dst_crs: CRS) -> Path: +def _warp_single( + input_path: Path, output_path: Path, dst_transform: Affine, width: int, height: int, dst_crs: CRS +) -> Path: resampling = _get_resampling_method(input_path) with rasterio.open(input_path) as src: @@ -191,15 +183,17 @@ def _warp_single(input_path: Path, output_path: Path, dst_transform: Affine, wid ) out_meta = src.meta.copy() - out_meta.update({ - "driver": "GTiff", - "height": height, - "width": width, - "transform": dst_transform, - "crs": dst_crs, - }) - - with rasterio.open(output_path, "w", **out_meta) as dest: + out_meta.update( + { + 'driver': 'GTiff', + 'height': height, + 'width': width, + 'transform': dst_transform, + 'crs': dst_crs, + } + ) + + with rasterio.open(output_path, 'w', **out_meta) as dest: dest.write(dst_data) return output_path @@ -208,14 +202,15 @@ def _warp_single(input_path: Path, output_path: Path, dst_transform: Affine, wid def _get_resampling_method(filepath: Path) -> Resampling: filename = filepath.name.lower() - if "fmask" in filename or "mask" in filename: + if 'fmask' in filename or 'mask' in filename: return Resampling.nearest else: return Resampling.bilinear - -def _build_common_grid(bounding_box_4326: tuple(float, float, float, float), reference_path: Path) -> tuple[Affine, int, int, CRS]: +def _build_common_grid( + bounding_box_4326: tuple(float, float, float, float), reference_path: Path +) -> tuple[Affine, int, int, CRS]: dst_crs = CRS.from_epsg(4326) minx, miny, maxx, maxy = bounding_box_4326 diff --git a/src/satchip/modality.py b/src/satchip/modality.py index 606b3d8..4f13ebe 100644 --- a/src/satchip/modality.py +++ b/src/satchip/modality.py @@ -1,10 +1,9 @@ from dataclasses import dataclass -from typing import Tuple @dataclass(frozen=True) class Modality: id: str - all_bands: Tuple[str, ...] - stack_bands: Tuple[str, ...] - chip_bands: Tuple[str, ...] + all_bands: tuple[str, ...] + stack_bands: tuple[str, ...] + chip_bands: tuple[str, ...] diff --git a/src/satchip/models.py b/src/satchip/models.py index 77d2fe5..a72984d 100644 --- a/src/satchip/models.py +++ b/src/satchip/models.py @@ -1,12 +1,11 @@ -from pathlib import Path -from datetime import datetime from dataclasses import dataclass -from typing import TypedDict, NamedTuple -from typing import Tuple +from datetime import datetime +from pathlib import Path +from typing import NamedTuple, TypedDict -from rasterio.coords import BoundingBox -import shapely import pyproj +import shapely +from rasterio.coords import BoundingBox class Band(NamedTuple): @@ -17,7 +16,7 @@ class Band(NamedTuple): class Modality(TypedDict): id: str - bands: Tuple[Band, ...] + bands: tuple[Band, ...] collection: str @@ -81,41 +80,41 @@ def bands_by_id(bands: tuple[Band, ...]) -> dict[str, Band]: MODALITIES: dict[str, Modality] = { - "OPERA_RTC": { - "id": "OPERA_RTC", - "collection": "OPERA_L2_RTC-S1_V1", - "bands": ( - Band("VV", "VV", "VV"), - Band("VH", "VH", "VH"), - Band("mask", "Validitiy Mask", "mask"), - ) + 'OPERA_RTC': { + 'id': 'OPERA_RTC', + 'collection': 'OPERA_L2_RTC-S1_V1', + 'bands': ( + Band('VV', 'VV', 'VV'), + Band('VH', 'VH', 'VH'), + Band('mask', 'Validitiy Mask', 'mask'), + ), + }, + 'HLS_S30': { + 'id': 'HLS_S30', + 'collection': 'HLSS30', + 'bands': ( + Band('B02', 'Blue', 'B'), + Band('B03', 'Green', 'G'), + Band('B04', 'Red', 'R'), + Band('B8A', 'NIR Narrow', 'N'), + Band('B11', 'SWIR 1', 'SW1'), + Band('B12', 'SWIR 2', 'SW2'), + Band('Fmask', 'Cloud Mask', 'Fmask'), + ), }, - "HLS_S30": { - "id": "HLS_S30", - "collection": "HLSS30", - "bands": ( - Band("B02", "Blue", "B"), - Band("B03", "Green", "G"), - Band("B04", "Red", "R"), - Band("B8A", "NIR Narrow", "N"), - Band("B11", "SWIR 1", "SW1"), - Band("B12", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "Fmask"), - ) + 'HLS_L30': { + 'id': 'HLS_L30', + 'collection': 'HLSL30', + 'bands': ( + Band('B02', 'Blue', 'B'), + Band('B03', 'Green', 'G'), + Band('B04', 'Red', 'R'), + Band('B05', 'NIR Narrow', 'N'), + Band('B06', 'SWIR 1', 'SW1'), + Band('B07', 'SWIR 2', 'SW2'), + Band('Fmask', 'Cloud Mask', 'fmask'), + ), }, - "HLS_L30": { - "id": "HLS_L30", - "collection": "HLSL30", - "bands": ( - Band("B02", "Blue", "B"), - Band("B03", "Green", "G"), - Band("B04", "Red", "R"), - Band("B05", "NIR Narrow", "N"), - Band("B06", "SWIR 1", "SW1"), - Band("B07", "SWIR 2", "SW2"), - Band("Fmask", "Cloud Mask", "fmask"), - ) - } } MODALITY_IDS = list(MODALITIES.keys()) diff --git a/src/satchip/mosaic.py b/src/satchip/mosaic.py index f42d73c..a5d1bc1 100644 --- a/src/satchip/mosaic.py +++ b/src/satchip/mosaic.py @@ -1,55 +1,51 @@ -from pathlib import Path -import shutil import datetime +import shutil +from pathlib import Path import earthaccess +import hls import numpy as np +import opera_rtc import rasterio +from modality import Modality from rasterio.crs import CRS from rasterio.merge import merge -from rasterio.warp import calculate_default_transform, reproject -from rasterio.warp import Resampling, transform_bounds from rasterio.transform import from_bounds - -from modality import Modality -import hls -import opera_rtc +from rasterio.warp import Resampling, calculate_default_transform, reproject, transform_bounds def data_over_swath(swath, modalities: list[Modality], output_path: Path): data_paths = { - "RAW": output_path / "RAW", - "REPROJECTED": output_path / "REPROJECTED", - "MOSAIC": output_path / "MOSAIC" + 'RAW': output_path / 'RAW', + 'REPROJECTED': output_path / 'REPROJECTED', + 'MOSAIC': output_path / 'MOSAIC', } - for p in ("MOSAIC", ): + for p in ('MOSAIC',): shutil.rmtree(data_paths[p], ignore_errors=True) for p in data_paths.values(): p.mkdir(parents=True, exist_ok=True) - swathID = f"{int(swath['swathID']):04d}" + swathID = f'{int(swath["swathID"]):04d}' stacked = {} for modality in modalities: print(f'Localizing data for {modality.id}.') - bounding_box = swath["buffered_event_background"].bounds + bounding_box = swath['buffered_event_background'].bounds - start_date = swath["ls5hlsDate"] if modality.id == 'HLS' else swath["s1Date"] + start_date = swath['ls5hlsDate'] if modality.id == 'HLS' else swath['s1Date'] results = search_data(bounding_box, start_date, modality) - local_files = earthaccess.download( - results, local_path=data_paths["RAW"], show_progress=True - ) + local_files = earthaccess.download(results, local_path=data_paths['RAW'], show_progress=True) - data_tifs = [f for f in local_files if f.name.endswith(".tif")] - reprojected_tifs = _reproject_files(data_tifs, output_path=data_paths["REPROJECTED"]) + data_tifs = [f for f in local_files if f.name.endswith('.tif')] + reprojected_tifs = _reproject_files(data_tifs, output_path=data_paths['REPROJECTED']) if len(data_tifs) == 0: - print(f"Skipping: no data for swath {swathID}") + print(f'Skipping: no data for swath {swathID}') continue mod_merged = {} @@ -58,9 +54,7 @@ def data_over_swath(swath, modalities: list[Modality], output_path: Path): band_files = [f for f in reprojected_tifs if band in band_from_filename(f.name, modality)] merged_name = make_merge_name(swathID, start_date, band, modality) - merged_band_path = _merge( - band_files, output_file=data_paths['MOSAIC'] / merged_name - ) + merged_band_path = _merge(band_files, output_file=data_paths['MOSAIC'] / merged_name) mod_merged[band] = merged_band_path @@ -104,19 +98,14 @@ def make_merge_name(swathID: str, start_date: datetime.datetime, band: str, moda return f'{swathID}.{modality.id}.{date_str}.{band}.tif' -def _reproject_files( - files: list[Path], output_path: Path -) -> list[Path]: - - reprojected_paths = [ - output_path / f"{granule.name}" for granule in files - ] +def _reproject_files(files: list[Path], output_path: Path) -> list[Path]: + reprojected_paths = [output_path / f'{granule.name}' for granule in files] for granule, output_path in zip(files, reprojected_paths): if output_path.exists(): continue - print(f"reprojecting to wgs84: {output_path.name}") + print(f'reprojecting to wgs84: {output_path.name}') _reproject_file(granule, output_path) return reprojected_paths @@ -126,16 +115,12 @@ def _reproject_file(local_file: Path, reprojected_file: Path, epsg=4326) -> None # https://rasterio.readthedocs.io/en/stable/topics/reproject.html#reprojecting-a-geotiff-dataset with rasterio.open(local_file) as src: dst_crs = CRS.from_epsg(epsg) - transform, width, height = calculate_default_transform( - src.crs, dst_crs, src.width, src.height, *src.bounds - ) + transform, width, height = calculate_default_transform(src.crs, dst_crs, src.width, src.height, *src.bounds) dst_kwargs = src.meta.copy() - dst_kwargs.update( - {"crs": dst_crs, "transform": transform, "width": width, "height": height} - ) + dst_kwargs.update({'crs': dst_crs, 'transform': transform, 'width': width, 'height': height}) - with rasterio.open(reprojected_file, "w", **dst_kwargs) as dst: + with rasterio.open(reprojected_file, 'w', **dst_kwargs) as dst: for i in range(1, src.count + 1): reproject( source=rasterio.band(src, i), @@ -163,15 +148,15 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: out_meta.update( { - "driver": "GTiff", - "height": mosaic.shape[0], - "width": mosaic.shape[1], - "transform": out_trans, - "crs": band_datasets[0].crs, + 'driver': 'GTiff', + 'height': mosaic.shape[0], + 'width': mosaic.shape[1], + 'transform': out_trans, + 'crs': band_datasets[0].crs, } ) - with rasterio.open(output_file, "w", **out_meta) as dst: + with rasterio.open(output_file, 'w', **out_meta) as dst: dst.write(mosaic, 1) finally: for ds in band_datasets: @@ -186,12 +171,11 @@ def _stack_bands(merged: dict[str, Path], data_bands: tuple[str], stacked_name: band = data_bands[0] meta.update(count=len(data_bands), dtype=np.float32) - stacked_file_name = _rename(merged[band], f"{band}.tif", f"{stacked_name}.tif") + stacked_file_name = _rename(merged[band], f'{band}.tif', f'{stacked_name}.tif') - with rasterio.open(stacked_file_name, "w", **meta) as dst: + with rasterio.open(stacked_file_name, 'w', **meta) as dst: for idx, band in enumerate(data_bands, start=1): with rasterio.open(merged[band]) as src: - dst.write(src.read(1), idx) merged[stacked_name] = stacked_file_name @@ -207,27 +191,21 @@ def _warp_over_swath(data, bounding_box_4326, output_dir): output_dir.mkdir(parents=True, exist_ok=True) reference_path = next(iter(next(iter(data.values())).values())) - dst_transform, width, height, dst_crs = _build_common_grid( - bounding_box_4326, reference_path - ) + dst_transform, width, height, dst_crs = _build_common_grid(bounding_box_4326, reference_path) output = {} for sensor, bands in data.items(): output[sensor] = {} for band_name, input_path in bands.items(): out_path = output_dir / input_path.name - _warp_single( - input_path, out_path, - dst_transform, width, height, dst_crs, - band_name - ) + _warp_single(input_path, out_path, dst_transform, width, height, dst_crs, band_name) output[sensor][band_name] = out_path return output def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, band_name): - CATEGORICAL_BANDS = {"Fmask", "mask"} + CATEGORICAL_BANDS = {'Fmask', 'mask'} resampling = Resampling.nearest if band_name in CATEGORICAL_BANDS else Resampling.bilinear with rasterio.open(input_path) as src: @@ -245,15 +223,17 @@ def _warp_single(input_path, output_path, dst_transform, width, height, dst_crs, ) out_meta = src.meta.copy() - out_meta.update({ - "driver": "GTiff", - "height": height, - "width": width, - "transform": dst_transform, - "crs": dst_crs, - }) - - with rasterio.open(output_path, "w", **out_meta) as dest: + out_meta.update( + { + 'driver': 'GTiff', + 'height': height, + 'width': width, + 'transform': dst_transform, + 'crs': dst_crs, + } + ) + + with rasterio.open(output_path, 'w', **out_meta) as dest: dest.write(dst_data) return output_path @@ -268,17 +248,17 @@ def _build_common_grid(bounding_box_4326, reference_path): ref_width = ref.width ref_height = ref.height - ref_bbox_width = bounds_4326[2] - bounds_4326[0] + ref_bbox_width = bounds_4326[2] - bounds_4326[0] ref_bbox_height = bounds_4326[3] - bounds_4326[1] - res_x = ref_bbox_width / ref_width + res_x = ref_bbox_width / ref_width res_y = ref_bbox_height / ref_height minx = np.floor(minx / res_x) * res_x miny = np.floor(miny / res_y) * res_y - maxx = np.ceil(maxx / res_x) * res_x - maxy = np.ceil(maxy / res_y) * res_y + maxx = np.ceil(maxx / res_x) * res_x + maxy = np.ceil(maxy / res_y) * res_y - width = int(round((maxx - minx) / res_x)) + width = int(round((maxx - minx) / res_x)) height = int(round((maxy - miny) / res_y)) dst_transform = from_bounds(minx, miny, maxx, maxy, width, height) diff --git a/src/satchip/old/chip_data.py b/src/satchip/old/chip_data.py index 496d7d6..ec13168 100644 --- a/src/satchip/old/chip_data.py +++ b/src/satchip/old/chip_data.py @@ -7,15 +7,15 @@ import earthaccess import numpy as np import xarray as xr -from shapely.geometry import box -from tqdm import tqdm - -from satchip import utils from satchip.chip_hls import get_hls_data from satchip.chip_hyp3s1rtc import get_rtc_paths_for_chips, get_s1rtc_chip_data from satchip.chip_operas1rtc import get_operartc_data from satchip.chip_sentinel2 import get_s2l2a_data from satchip.terra_mind_grid import TerraMindChip, TerraMindGrid +from shapely.geometry import box +from tqdm import tqdm + +from satchip import utils def fill_missing_times(data_chip: xr.DataArray, times: np.ndarray) -> xr.DataArray: diff --git a/src/satchip/old/chip_hls.py b/src/satchip/old/chip_hls.py index b6fb9b8..5163f0d 100644 --- a/src/satchip/old/chip_hls.py +++ b/src/satchip/old/chip_hls.py @@ -8,11 +8,11 @@ import shapely import xarray as xr from earthaccess.results import DataGranule +from satchip.chip_xr_base import create_dataset_chip, create_template_da +from satchip.terra_mind_grid import TerraMindChip from shapely.geometry import Polygon from satchip import utils -from satchip.chip_xr_base import create_dataset_chip, create_template_da -from satchip.terra_mind_grid import TerraMindChip earthaccess.login() diff --git a/src/satchip/old/chip_hyp3s1rtc.py b/src/satchip/old/chip_hyp3s1rtc.py index f94625c..c349737 100644 --- a/src/satchip/old/chip_hyp3s1rtc.py +++ b/src/satchip/old/chip_hyp3s1rtc.py @@ -7,11 +7,11 @@ import rioxarray import shapely import xarray as xr - -from satchip import utils from satchip.chip_xr_base import create_dataset_chip, create_template_da from satchip.terra_mind_grid import TerraMindChip +from satchip import utils + S1RTC_BANDS = ('VV', 'VH') diff --git a/src/satchip/old/chip_label.py b/src/satchip/old/chip_label.py index 246f8e0..6234a9f 100644 --- a/src/satchip/old/chip_label.py +++ b/src/satchip/old/chip_label.py @@ -5,11 +5,11 @@ import numpy as np import rasterio as rio import xarray as xr +from satchip.chip_xr_base import create_dataset_chip +from satchip.terra_mind_grid import TerraMindGrid from tqdm import tqdm from satchip import utils -from satchip.chip_xr_base import create_dataset_chip -from satchip.terra_mind_grid import TerraMindGrid def is_valuable(chip: np.ndarray) -> bool: diff --git a/src/satchip/old/chip_operas1rtc.py b/src/satchip/old/chip_operas1rtc.py index 055232e..4eb8151 100644 --- a/src/satchip/old/chip_operas1rtc.py +++ b/src/satchip/old/chip_operas1rtc.py @@ -8,12 +8,12 @@ import xarray as xr from earthaccess.results import DataGranule from osgeo import gdal - -from satchip import utils from satchip.chip_hls import get_geometry, get_product_id from satchip.chip_xr_base import create_dataset_chip, create_template_da from satchip.terra_mind_grid import TerraMindChip +from satchip import utils + gdal.UseExceptions() diff --git a/src/satchip/old/chip_sentinel2.py b/src/satchip/old/chip_sentinel2.py index 34ae90c..39370ed 100644 --- a/src/satchip/old/chip_sentinel2.py +++ b/src/satchip/old/chip_sentinel2.py @@ -10,11 +10,11 @@ import xarray as xr from pystac.item import Item from pystac_client import Client - -from satchip import utils from satchip.chip_xr_base import create_dataset_chip, create_template_da from satchip.terra_mind_grid import TerraMindChip +from satchip import utils + S2_BANDS = OrderedDict( { diff --git a/src/satchip/old/chip_xr_base.py b/src/satchip/old/chip_xr_base.py index b441ef1..e9541e2 100644 --- a/src/satchip/old/chip_xr_base.py +++ b/src/satchip/old/chip_xr_base.py @@ -3,9 +3,9 @@ import numpy as np import xarray as xr +from satchip.terra_mind_grid import TerraMindChip import satchip -from satchip.terra_mind_grid import TerraMindChip def _check_spec(dataset: xr.Dataset) -> None: diff --git a/src/satchip/old/terra_mind_grid.py b/src/satchip/old/terra_mind_grid.py index 71596b6..cc4876d 100644 --- a/src/satchip/old/terra_mind_grid.py +++ b/src/satchip/old/terra_mind_grid.py @@ -3,8 +3,8 @@ import numpy as np import pyproj from rasterio import Affine - from satchip.major_tom_grid import MajorTomGrid + from satchip.utils import get_epsg4326_bbox, get_epsg4326_point diff --git a/src/satchip/old/utils.py b/src/satchip/old/utils.py index e4ffd10..fd2467b 100644 --- a/src/satchip/old/utils.py +++ b/src/satchip/old/utils.py @@ -91,12 +91,9 @@ def load_chip(label_path: str | Path) -> xr.Dataset: def retry_on_connection_error( - max_retries: int = 3, - backoff_factor: float = 1 + max_retries: int = 3, backoff_factor: float = 1 ) -> Callable[[Callable[P, R]], Callable[P, R]]: - def decorator(func: Callable[P, R]) -> Callable[P, R]: - @functools.wraps(func) def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: for attempt in range(max_retries): @@ -104,13 +101,14 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: return func(*args, **kwargs) except (ConnectionError, OSError) as e: if attempt == max_retries - 1: - print(f"Failed after {max_retries} attempts: {e}") + print(f'Failed after {max_retries} attempts: {e}') raise e - wait_time = backoff_factor * (2 ** attempt) + wait_time = backoff_factor * (2**attempt) time.sleep(wait_time) - raise RuntimeError("Unexpected exit from retry loop") + raise RuntimeError('Unexpected exit from retry loop') return wrapper + return decorator diff --git a/src/satchip/opera_rtc.py b/src/satchip/opera_rtc.py index cf54737..f67768d 100644 --- a/src/satchip/opera_rtc.py +++ b/src/satchip/opera_rtc.py @@ -1,18 +1,18 @@ -from datetime import timedelta, datetime +from datetime import datetime, timedelta from pathlib import Path import earthaccess -from earthaccess.results import DataGranule import numpy as np import rasterio +from earthaccess.results import DataGranule def search_rtc_data(start_date: datetime, bounding_box: tuple[float, float, float, float]) -> list[DataGranule]: final_date = start_date + timedelta(days=1) results = earthaccess.search_data( - short_name=["OPERA_L2_RTC-S1_V1"], - temporal=(start_date.strftime("%Y-%m-%d"), final_date.strftime("%Y-%m-%d")), + short_name=['OPERA_L2_RTC-S1_V1'], + temporal=(start_date.strftime('%Y-%m-%d'), final_date.strftime('%Y-%m-%d')), bounding_box=bounding_box, ) @@ -34,12 +34,12 @@ def make_merged_rtc_name(template_filename: str) -> str: """ # ['1442.OPERA', 'L2', 'RTC-S1', 'T063-133415-IW2', '20170620T001327Z', '20250925T045340Z', 'S1A', '30', 'v1.0', 'VV.tif'] - name_parts = template_filename.split("_") + name_parts = template_filename.split('_') name_parts.pop(5) # Remove Product Generation Time name_parts.pop(3) # Remove Burst ID - return "_".join(name_parts) + return '_'.join(name_parts) def is_valid_rtc(mask_path: Path, label_path: Path) -> bool: @@ -57,7 +57,7 @@ def is_valid_rtc(mask_path: Path, label_path: Path) -> bool: valid_event_pixels = (is_event_pixel & is_valid_pixel).sum() pct_valid_data = 100.0 * valid_event_pixels / total_event_pixels - print(f"Percent of the event with valid data: {pct_valid_data:.1f}%") + print(f'Percent of the event with valid data: {pct_valid_data:.1f}%') return pct_valid_data > 50.0 @@ -66,10 +66,10 @@ def filter_rtc_chips(chips: dict[str, dict]) -> list[dict]: good_chips = [] for tile_id, chip in chips.items(): - with rasterio.open(chip["BANDS"]) as ds: + with rasterio.open(chip['BANDS']) as ds: rtc_data = ds.read() - with rasterio.open(chip["EVENT"]) as ds: + with rasterio.open(chip['EVENT']) as ds: event_mask = ds.read(1) has_nan_pixels = np.isnan(rtc_data).sum() > 0 @@ -86,9 +86,7 @@ def filter_rtc_chips(chips: dict[str, dict]) -> list[dict]: return good_chips -def normalize_image_array( - input_array: np.ndarray, vmin: float, vmax: float -) -> np.ndarray: +def normalize_image_array(input_array: np.ndarray, vmin: float, vmax: float) -> np.ndarray: input_array = input_array.astype(float) scaled_array = (input_array - vmin) / (vmax - vmin) scaled_array[np.isnan(input_array)] = 0 diff --git a/tests/conftest.py b/tests/conftest.py index 1ed5ba7..868682c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,46 +1,47 @@ from datetime import datetime from pathlib import Path -import pytest import geopandas as gpd import pandas as pd +import pytest from shapely import wkt from satchip import download_data, models -DATA_PATH = Path(__file__).parent / "data" + +DATA_PATH = Path(__file__).parent / 'data' def pytest_addoption(parser): - parser.addoption("--download", action="store_true", default=False) + parser.addoption('--download', action='store_true', default=False) def pytest_configure(config): - config.addinivalue_line("markers", "download: marks tests that download data") + config.addinivalue_line('markers', 'download: marks tests that download data') def pytest_collection_modifyitems(config, items): - if not config.getoption("--download"): - skip = pytest.mark.skip(reason="pass --download to run") + if not config.getoption('--download'): + skip = pytest.mark.skip(reason='pass --download to run') for item in items: - if item.get_closest_marker("download"): + if item.get_closest_marker('download'): item.add_marker(skip) @pytest.fixture def pristine_gdf(tmp_path): - shp_path = DATA_PATH / "hwds_pristine" + shp_path = DATA_PATH / 'hwds_pristine' df = gpd.read_file(shp_path) - df["SwathDate"] = pd.to_datetime(df["SwathDate"], format="%Y-%m-%d") - df["HLSDate"] = pd.to_datetime(df["HLSDate"], format="%Y-%m-%d") + df['SwathDate'] = pd.to_datetime(df['SwathDate'], format='%Y-%m-%d') + df['HLSDate'] = pd.to_datetime(df['HLSDate'], format='%Y-%m-%d') return df -@pytest.fixture(scope="session") +@pytest.fixture(scope='session') def warped_event_files(): - warped_base = DATA_PATH / "warped" + warped_base = DATA_PATH / 'warped' return [ warped_base / '102a.HLS_S30.2019-07-09.Fmask.tif', @@ -49,9 +50,9 @@ def warped_event_files(): ] -@pytest.fixture(scope="session") +@pytest.fixture(scope='session') def s2_local_files(s2_event): - download_path = DATA_PATH / "raw" + download_path = DATA_PATH / 'raw' print('Downloading test data...') local_files = download_data.download_data(s2_event, models.HLS_S30, download_path) @@ -59,9 +60,11 @@ def s2_local_files(s2_event): return local_files -@pytest.fixture(scope="session") +@pytest.fixture(scope='session') def s2_event() -> models.Event: - geom = wkt.loads('POLYGON Z ((-97.46296108799999 41.72598898100006 0, -97.45654663199997 41.72832979400005 0, -97.45286506599996 41.735911517000034 0, -97.44692854399995 41.73806293200005 0, -97.44255668499994 41.741031193000026 0, -97.43649360699999 41.746392472000025 0, -97.43643518499994 41.74966361400004 0, -97.44477798299994 41.751269181000055 0, -97.44546922799998 41.757304729000055 0, -97.44413504799996 41.76092731800003 0, -97.43601483299994 41.761099638000076 0, -97.43493753799999 41.76400779800008 0, -97.43574582899998 41.76925407400006 0, -97.43507482199999 41.775018801000044 0, -97.42929198599995 41.77529913700005 0, -97.42434097299997 41.77189406900004 0, -97.41643891099994 41.77095250000008 0, -97.40855823399994 41.772049811000045 0, -97.40518880099995 41.769938841000055 0, -97.40212726299995 41.772478093000075 0, -97.39708935799996 41.77557899800007 0, -97.39216022699998 41.77683695500008 0, -97.38494019299998 41.779687625000065 0, -97.37802386699997 41.78005042400008 0, -97.37595922599996 41.772699628000055 0, -97.36870573799996 41.772803176000025 0, -97.36473372899997 41.776025134000065 0, -97.35972290999996 41.77556353600005 0, -97.35569283499996 41.77524485300006 0, -97.35141833499995 41.77176253300007 0, -97.35179576499996 41.769616866000035 0, -97.35247585899998 41.76537635100004 0, -97.35455601099994 41.765104164000036 0, -97.35748216699994 41.763087014000064 0, -97.35755091099998 41.760709776000056 0, -97.35659214299994 41.75815047000003 0, -97.34941309099997 41.75410074900003 0, -97.34425890099999 41.75299627900006 0, -97.33955929299998 41.75280198300004 0, -97.33427827799994 41.75626863400004 0, -97.32638406299998 41.75688761500004 0, -97.32401029299996 41.75030413500008 0, -97.32617321299995 41.74390741500008 0, -97.33893306699997 41.74281858000006 0, -97.33915551999996 41.73881615700003 0, -97.33542314499994 41.73642923700004 0, -97.33487090999995 41.73053873300006 0, -97.33494400699999 41.72165371600005 0, -97.34530133099997 41.72321647500007 0, -97.34362613199994 41.72861741600008 0, -97.34850260899998 41.72835093900005 0, -97.34904493499994 41.72262336800003 0, -97.34959717099997 41.71581247300003 0, -97.35659214299994 41.713051300000075 0, -97.35972147399997 41.70716079600004 0, -97.36818907299994 41.707897109000044 0, -97.37554466699999 41.70552724600003 0, -97.37794521999996 41.71268314300005 0, -97.38066826999994 41.713925654000036 0, -97.38197931699995 41.71100923000006 0, -97.38053398199997 41.702322166000044 0, -97.37438188299996 41.70104730000003 0, -97.37256136399998 41.69851681000006 0, -97.37141382499999 41.691215095000075 0, -97.37359654199997 41.685445995000066 0, -97.37716937399995 41.686764063000055 0, -97.37836702299995 41.692105638000044 0, -97.38678588699997 41.69234998300004 0, -97.38711030299999 41.69474924900004 0, -97.38893657299997 41.696020211000075 0, -97.39203310699997 41.69605112600004 0, -97.39708425799995 41.69660031600006 0, -97.39699799999994 41.69790345900003 0, -97.39728821299997 41.70098871600004 0, -97.39958472399996 41.70204029100006 0, -97.40051777999997 41.703388508000046 0, -97.40357986399994 41.71025483200003 0, -97.40555695699999 41.71360353400007 0, -97.41463431499994 41.71473101400005 0, -97.41599053299996 41.71041910900004 0, -97.40847919899994 41.70361729000007 0, -97.40905360899995 41.698218834000045 0, -97.42033287699996 41.69235264400004 0, -97.42815022299999 41.69435335000003 0, -97.42415654299998 41.69969951000007 0, -97.42249215599998 41.70624040400003 0, -97.42727819099997 41.70936973400006 0, -97.43331385799996 41.70941182400003 0, -97.43942735499996 41.70716079600004 0, -97.44623824999996 41.70182252600006 0, -97.45065612799999 41.700902135000035 0, -97.45614052299999 41.70244454500005 0, -97.45855998799999 41.70632093900008 0, -97.46176883199996 41.71106968300006 0, -97.46907545699997 41.71252207500004 0, -97.47778018699995 41.71142744600007 0, -97.48360613399996 41.71139459500006 0, -97.48556606699998 41.71563875600003 0, -97.47960867399996 41.71848116100006 0, -97.47027376699998 41.720714549000036 0, -97.46593462299995 41.726661585000045 0, -97.46296108799999 41.72598898100006 0))') + geom = wkt.loads( + 'POLYGON Z ((-97.46296108799999 41.72598898100006 0, -97.45654663199997 41.72832979400005 0, -97.45286506599996 41.735911517000034 0, -97.44692854399995 41.73806293200005 0, -97.44255668499994 41.741031193000026 0, -97.43649360699999 41.746392472000025 0, -97.43643518499994 41.74966361400004 0, -97.44477798299994 41.751269181000055 0, -97.44546922799998 41.757304729000055 0, -97.44413504799996 41.76092731800003 0, -97.43601483299994 41.761099638000076 0, -97.43493753799999 41.76400779800008 0, -97.43574582899998 41.76925407400006 0, -97.43507482199999 41.775018801000044 0, -97.42929198599995 41.77529913700005 0, -97.42434097299997 41.77189406900004 0, -97.41643891099994 41.77095250000008 0, -97.40855823399994 41.772049811000045 0, -97.40518880099995 41.769938841000055 0, -97.40212726299995 41.772478093000075 0, -97.39708935799996 41.77557899800007 0, -97.39216022699998 41.77683695500008 0, -97.38494019299998 41.779687625000065 0, -97.37802386699997 41.78005042400008 0, -97.37595922599996 41.772699628000055 0, -97.36870573799996 41.772803176000025 0, -97.36473372899997 41.776025134000065 0, -97.35972290999996 41.77556353600005 0, -97.35569283499996 41.77524485300006 0, -97.35141833499995 41.77176253300007 0, -97.35179576499996 41.769616866000035 0, -97.35247585899998 41.76537635100004 0, -97.35455601099994 41.765104164000036 0, -97.35748216699994 41.763087014000064 0, -97.35755091099998 41.760709776000056 0, -97.35659214299994 41.75815047000003 0, -97.34941309099997 41.75410074900003 0, -97.34425890099999 41.75299627900006 0, -97.33955929299998 41.75280198300004 0, -97.33427827799994 41.75626863400004 0, -97.32638406299998 41.75688761500004 0, -97.32401029299996 41.75030413500008 0, -97.32617321299995 41.74390741500008 0, -97.33893306699997 41.74281858000006 0, -97.33915551999996 41.73881615700003 0, -97.33542314499994 41.73642923700004 0, -97.33487090999995 41.73053873300006 0, -97.33494400699999 41.72165371600005 0, -97.34530133099997 41.72321647500007 0, -97.34362613199994 41.72861741600008 0, -97.34850260899998 41.72835093900005 0, -97.34904493499994 41.72262336800003 0, -97.34959717099997 41.71581247300003 0, -97.35659214299994 41.713051300000075 0, -97.35972147399997 41.70716079600004 0, -97.36818907299994 41.707897109000044 0, -97.37554466699999 41.70552724600003 0, -97.37794521999996 41.71268314300005 0, -97.38066826999994 41.713925654000036 0, -97.38197931699995 41.71100923000006 0, -97.38053398199997 41.702322166000044 0, -97.37438188299996 41.70104730000003 0, -97.37256136399998 41.69851681000006 0, -97.37141382499999 41.691215095000075 0, -97.37359654199997 41.685445995000066 0, -97.37716937399995 41.686764063000055 0, -97.37836702299995 41.692105638000044 0, -97.38678588699997 41.69234998300004 0, -97.38711030299999 41.69474924900004 0, -97.38893657299997 41.696020211000075 0, -97.39203310699997 41.69605112600004 0, -97.39708425799995 41.69660031600006 0, -97.39699799999994 41.69790345900003 0, -97.39728821299997 41.70098871600004 0, -97.39958472399996 41.70204029100006 0, -97.40051777999997 41.703388508000046 0, -97.40357986399994 41.71025483200003 0, -97.40555695699999 41.71360353400007 0, -97.41463431499994 41.71473101400005 0, -97.41599053299996 41.71041910900004 0, -97.40847919899994 41.70361729000007 0, -97.40905360899995 41.698218834000045 0, -97.42033287699996 41.69235264400004 0, -97.42815022299999 41.69435335000003 0, -97.42415654299998 41.69969951000007 0, -97.42249215599998 41.70624040400003 0, -97.42727819099997 41.70936973400006 0, -97.43331385799996 41.70941182400003 0, -97.43942735499996 41.70716079600004 0, -97.44623824999996 41.70182252600006 0, -97.45065612799999 41.700902135000035 0, -97.45614052299999 41.70244454500005 0, -97.45855998799999 41.70632093900008 0, -97.46176883199996 41.71106968300006 0, -97.46907545699997 41.71252207500004 0, -97.47778018699995 41.71142744600007 0, -97.48360613399996 41.71139459500006 0, -97.48556606699998 41.71563875600003 0, -97.47960867399996 41.71848116100006 0, -97.47027376699998 41.720714549000036 0, -97.46593462299995 41.726661585000045 0, -97.46296108799999 41.72598898100006 0))' + ) name = '95a' date = datetime(year=2018, month=7, day=2) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index e46c856..f1267e0 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -1,7 +1,6 @@ import pytest -from satchip import models -from satchip import download_data +from satchip import download_data, models @pytest.mark.download diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 9e9a10c..7924506 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,6 +1,7 @@ -from satchip import merge_modality, models, generate_labels import rasterio +from satchip import generate_labels, merge_modality, models + def test_make_merge_name(s2_event): merged_name = merge_modality._make_merge_name(s2_event.name, s2_event.date, 'BAND', 'MOD') @@ -54,7 +55,9 @@ def test_warp_to_reference(s2_local_files, s2_event, tmp_path): stacked = merge_modality.stack_bands(bands, stacked_filename=tmp_path / 'stacked.tif') label = generate_labels.binary_mask_from_template(stacked, s2_event, tmp_path) - outputs = merge_modality.warp_to_reference(label, [stacked, fmask], tmp_path / 'warped', s2_event.buffered_geometry().bounds) + outputs = merge_modality.warp_to_reference( + label, [stacked, fmask], tmp_path / 'warped', s2_event.buffered_geometry().bounds + ) shapes = set() for output in outputs: diff --git a/tests/test_models.py b/tests/test_models.py index 116067e..49a2d83 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -14,36 +14,42 @@ def test_hls_model_names(): assert models.HLS_S30_BANDS['SW2'].id != models.HLS_L30_BANDS['SW2'].id -@pytest.mark.parametrize("filename, expected_band", [ +@pytest.mark.parametrize( + 'filename, expected_band', + [ ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_VH.tif', 'VH'), ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_VH.tif', 'VH'), ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_VV.tif', 'VV'), ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_VV.tif', 'VV'), ('OPERA_L2_RTC-S1_T085-181260-IW1_20190822T125312Z_20250913T222203Z_S1A_30_v1.0_mask.tif', 'mask'), ('OPERA_L2_RTC-S1_T165-352512-IW3_20200723T000516Z_20250908T213809Z_S1B_30_v1.0_mask.tif', 'mask'), -]) + ], +) def test_band_id_from_filename_rtc(filename, expected_band): assert models.band_id_from_filename(filename, 'OPERA_RTC').id == expected_band -@pytest.mark.parametrize("filename, expected_band", [ - ('HLS.S30.T13TFL.2019187T174919.v2.0.B12.tif', 'B12'), - ('HLS.S30.T13TGM.2018184T173901.v2.0.B11.tif', 'B11'), - ('HLS.S30.T14TLQ.2019219T173911.v2.0.B8A.tif', 'B8A'), - ('HLS.S30.T15TXG.2017167T170311.v2.0.B04.tif', 'B04'), - ('HLS.S30.T13TFL.2019187T174919.v2.0.Fmask.tif', 'Fmask'), - ('HLS.S30.T13TGM.2018184T173901.v2.0.B12.tif', 'B12'), - ('HLS.S30.T14TLQ.2019219T173911.v2.0.B11.tif', 'B11'), - ('HLS.S30.T15TXG.2017167T170311.v2.0.B8A.tif', 'B8A'), - ('HLS.S30.T13TFL.2020157T174911.v2.0.B02.tif', 'B02'), - ('HLS.S30.T13TGM.2018184T173901.v2.0.Fmask.tif', 'Fmask'), - ('HLS.S30.T13TFL.2020157T174911.v2.0.B03.tif', 'B03'), - ('HLS.S30.T14SKH.2019211T172909.v2.0.B02.tif', 'B02'), - ('HLS.S30.T14TLQ.2019219T173911.v2.0.Fmask.tif', 'Fmask'), - ('HLS.S30.T13TFL.2020157T174911.v2.0.B04.tif', 'B04'), - ('HLS.S30.T14SKH.2019211T172909.v2.0.B03.tif', 'B03'), - ('HLS.S30.T14TLQ.2020156T172859.v2.0.B02.tif', 'B02'), - ('HLS.S30.T15TXG.2017167T170311.v2.0.Fmask.tif', 'Fmask'), -]) +@pytest.mark.parametrize( + 'filename, expected_band', + [ + ('HLS.S30.T13TFL.2019187T174919.v2.0.B12.tif', 'B12'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.B11.tif', 'B11'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.B8A.tif', 'B8A'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.B04.tif', 'B04'), + ('HLS.S30.T13TFL.2019187T174919.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.B12.tif', 'B12'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.B11.tif', 'B11'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.B8A.tif', 'B8A'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B02.tif', 'B02'), + ('HLS.S30.T13TGM.2018184T173901.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B03.tif', 'B03'), + ('HLS.S30.T14SKH.2019211T172909.v2.0.B02.tif', 'B02'), + ('HLS.S30.T14TLQ.2019219T173911.v2.0.Fmask.tif', 'Fmask'), + ('HLS.S30.T13TFL.2020157T174911.v2.0.B04.tif', 'B04'), + ('HLS.S30.T14SKH.2019211T172909.v2.0.B03.tif', 'B03'), + ('HLS.S30.T14TLQ.2020156T172859.v2.0.B02.tif', 'B02'), + ('HLS.S30.T15TXG.2017167T170311.v2.0.Fmask.tif', 'Fmask'), + ], +) def test_band_id_from_filename_hls_s30(filename, expected_band): assert models.band_id_from_filename(filename, 'HLS_S30').id == expected_band From 058a7d24e8474f7884764714b957df52423ef9fc Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 7 Apr 2026 08:05:44 -0800 Subject: [PATCH 18/21] Basic view chips function --- notebooks/hwds-example.ipynb | 109 ++++++++++++---- src/satchip/merge_modality.py | 12 +- src/satchip/view.py | 229 ++++++++++++++++++++++++++++++++++ tests/test_merge_modality.py | 22 ++-- tests/test_view.py | 41 ++++++ 5 files changed, 377 insertions(+), 36 deletions(-) create mode 100644 src/satchip/view.py create mode 100644 tests/test_view.py diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index 6c7b248..d51e4d1 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "4dcd6420", "metadata": {}, "outputs": [ @@ -21,12 +21,12 @@ "import geopandas as gpd\n", "import pandas as pd\n", "\n", - "from satchip import models, download_data, merge_modality, generate_labels, chip_data" + "from satchip import models, download_data, merge_modality, generate_labels, chip_data, view" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "257c1816", "metadata": {}, "outputs": [], @@ -46,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "2b0886a1", "metadata": {}, "outputs": [], @@ -60,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "3dc85728", "metadata": {}, "outputs": [ @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "6a611425", "metadata": {}, "outputs": [ @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0d0202e9", "metadata": {}, "outputs": [ @@ -122,9 +122,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10394.81it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 230879.12it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 373749.86it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 9419.52it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 277564.24it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 354448.23it/s]\n" ] } ], @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "9b99ff34", "metadata": {}, "outputs": [], @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e6ce6b53", "metadata": {}, "outputs": [ @@ -171,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "9673d689", "metadata": {}, "outputs": [], @@ -184,7 +184,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "e6f7e30b", "metadata": {}, "outputs": [], @@ -194,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9d9b6664", "metadata": {}, "outputs": [ @@ -212,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "3180399a", "metadata": {}, "outputs": [], @@ -227,7 +227,58 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, + "id": "7e3e9a4b", + "metadata": {}, + "outputs": [], + "source": [ + "import rasterio\n", + "\n", + "with rasterio.open(bands) as ds:\n", + " bounds = ds.bounds\n", + " full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top]\n", + "\n", + " band_data = ds.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9513cb17", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Expected a projection subclass. Cannot handle a in imshow.", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m view.view_merged(bands, event, modality, [\u001b[32m2\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m0\u001b[39m], quite=\u001b[38;5;28;01mFalse\u001b[39;00m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Repositories/fm/satchip/src/satchip/view.py:41\u001b[39m, in \u001b[36mview_merged\u001b[39m\u001b[34m(stacked_data_file, event, modality, rgb_bands, save_to_file, quite)\u001b[39m\n\u001b[32m 31\u001b[39m fig, ax = plt.subplots(\n\u001b[32m 32\u001b[39m \u001b[32m1\u001b[39m,\n\u001b[32m 33\u001b[39m \u001b[32m1\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 36\u001b[39m layout=\u001b[33m'\u001b[39m\u001b[33mconstrained\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m 37\u001b[39m )\n\u001b[32m 39\u001b[39m event_geom = event.wgs84_geometry\n\u001b[32m---> \u001b[39m\u001b[32m41\u001b[39m \u001b[43max\u001b[49m\u001b[43m.\u001b[49m\u001b[43mimshow\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextent\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfull_extent\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morigin\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mupper\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 42\u001b[39m ax.add_geometries([event_geom], edgecolor=\u001b[33m'\u001b[39m\u001b[33mred\u001b[39m\u001b[33m'\u001b[39m, linewidth=\u001b[32m2\u001b[39m, facecolor=\u001b[33m'\u001b[39m\u001b[33mnone\u001b[39m\u001b[33m'\u001b[39m, crs=crs_pc)\n\u001b[32m 44\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m save_to_file:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/miniforge3/envs/satchip/lib/python3.14/site-packages/cartopy/mpl/geoaxes.py:291\u001b[39m, in \u001b[36m_add_transform..wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 286\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mInvalid transform: Spherical \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc.\u001b[34m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 287\u001b[39m \u001b[33m'\u001b[39m\u001b[33mis not supported - consider using \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 288\u001b[39m \u001b[33m'\u001b[39m\u001b[33mPlateCarree/RotatedPole.\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 290\u001b[39m kwargs[\u001b[33m'\u001b[39m\u001b[33mtransform\u001b[39m\u001b[33m'\u001b[39m] = transform\n\u001b[32m--> \u001b[39m\u001b[32m291\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/miniforge3/envs/satchip/lib/python3.14/site-packages/cartopy/mpl/geoaxes.py:1287\u001b[39m, in \u001b[36mGeoAxes.imshow\u001b[39m\u001b[34m(self, img, *args, **kwargs)\u001b[39m\n\u001b[32m 1284\u001b[39m kwargs[\u001b[33m'\u001b[39m\u001b[33morigin\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m'\u001b[39m\u001b[33mlower\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1286\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(transform, ccrs.Projection):\n\u001b[32m-> \u001b[39m\u001b[32m1287\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m'\u001b[39m\u001b[33mExpected a projection subclass. Cannot \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1288\u001b[39m \u001b[33m'\u001b[39m\u001b[33mhandle a \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m in imshow.\u001b[39m\u001b[33m'\u001b[39m % \u001b[38;5;28mtype\u001b[39m(transform))\n\u001b[32m 1290\u001b[39m target_extent = \u001b[38;5;28mself\u001b[39m.get_extent(\u001b[38;5;28mself\u001b[39m.projection)\n\u001b[32m 1291\u001b[39m regrid_shape = kwargs.pop(\u001b[33m'\u001b[39m\u001b[33mregrid_shape\u001b[39m\u001b[33m'\u001b[39m, \u001b[32m750\u001b[39m)\n", + "\u001b[31mValueError\u001b[39m: Expected a projection subclass. Cannot handle a in imshow." + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABLsAAAJnCAYAAAB/KlGVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAEX9JREFUeJzt3LFq41AQQFFlCcKVazUq/P8f5cKNa1dGjYIWttysghU2XM6ph2Hqy+O9reu6DgAAAAAQ8Ot/HwAAAAAARxG7AAAAAMgQuwAAAADIELsAAAAAyBC7AAAAAMgQuwAAAADIELsAAAAAyHjfO/h8PodlWb73GgAAAAD4i3Ech9PpNLwcu7bQdblchvv9vmccAAAAAA43TdNwvV4/DV67Ytf2omsLXbfbbTifz0feCAAAAAD/9Hg8hnmef3eql2PXH1voErsAAAAA+Kl8UA8AAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAxvtXhh+Px/ddAgAAAAAvdqldsWscx2GapmGe511LAQAAAOBoW5/aOtVn3tZ1Xfcsez6fw7IsR90GAAAAAF+yha7T6XRM7AIAAACAn84H9QAAAABkiF0AAAAAZIhdAAAAAGSIXQAAAABkiF0AAAAAZIhdAAAAAGSIXQAAAAAMFR+6uDuDhX37uwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "view.view_merged(bands, event, modality, [2, 1, 0], quite=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "1f65b3ed", "metadata": {}, "outputs": [], @@ -237,7 +288,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "3c4cee14", "metadata": {}, "outputs": [], @@ -249,7 +300,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "8d3ac758", "metadata": {}, "outputs": [], @@ -260,7 +311,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "e5c082c9", "metadata": {}, "outputs": [ @@ -274,7 +325,7 @@ " ChipStack(id='001.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.005.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.005.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))})]" ] }, - "execution_count": 19, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -290,6 +341,22 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "549d0513", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70358b1c", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 0c71a81..4dd4b0e 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -35,7 +35,7 @@ def merge_modality( merged_name = _make_merge_name(event.name, event.date, band.shortname, modality['id']) - merged_band_path = _merge(band_files, output_file=output_path / merged_name) + merged_band_path = merge_files(band_files, output_file=output_path / merged_name) merged.append(merged_band_path) @@ -48,8 +48,8 @@ def _make_merge_name(event_name: str, start_date: datetime.datetime, band: str, return f'{event_name}.{modality_id}.{date_str}.{band}.tif' -def _merge(band_files: list[Path], output_file: Path) -> Path: - band_datasets = [rasterio.open(band_file) for band_file in band_files] +def merge_files(files: list[Path], output_file: Path) -> Path: + band_datasets = [rasterio.open(band_file) for band_file in files] reference_crs = band_datasets[0].crs for ds in band_datasets[1:]: @@ -73,7 +73,11 @@ def _merge(band_files: list[Path], output_file: Path) -> Path: ) with rasterio.open(output_file, 'w', **out_meta) as dst: - dst.write(mosaic, 1) + if len(mosaic.shape) == 2: + dst.write(mosaic, 1) + else: + dst.write(mosaic) + finally: for ds in band_datasets: ds.close() diff --git a/src/satchip/view.py b/src/satchip/view.py new file mode 100644 index 0000000..83d7378 --- /dev/null +++ b/src/satchip/view.py @@ -0,0 +1,229 @@ +from pathlib import Path + +import rasterio +from rasterio.merge import merge +import numpy as np +import matplotlib.pyplot as plt +import cartopy.crs as ccrs +from shapely.geometry import box + +from satchip import models, merge_modality + + +def view_merged( + stacked_data_file: Path, + event: models.Event, + modality: models.Modality, + rgb_bands: tuple[int, int, int], + save_to_file: Path | None = None, + quite: bool = False, +): + crs_pc = ccrs.PlateCarree() + + with rasterio.open(stacked_data_file) as ds: + bounds = ds.bounds + full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] + band_data = ds.read() + + img = get_img(band_data, modality, rgb_bands) + + # plot BANDS and geom + fig, ax = plt.subplots( + 1, + 1, + subplot_kw={'projection': crs_pc}, + figsize=(12, 12), + layout='constrained', + ) + + event_geom = event.wgs84_geometry + + ax.imshow(img, extent=full_extent, origin='upper', transform=crs_pc) + ax.add_geometries([event_geom], edgecolor='red', linewidth=2, facecolor='none', crs=crs_pc) + + if save_to_file: + plt.savefig( + save_to_file, + dpi=300, + bbox_inches='tight', + ) + + if not quite: + plt.show() + + plt.close(fig) + + +def view_chip( + chip: models.ChipStack, + modality: models.Modality, + rgb_bands: tuple[int, int, int], + save_to_file: Path | None = None, + quite: bool = False, +): + with rasterio.open(chip.data) as ds: + bounds = ds.bounds + full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] + band_data = ds.read() + + with rasterio.open(chip.label) as ds: + label_data = ds.read().squeeze() + + img = get_img(band_data, modality, rgb_bands) + + crs_pc = ccrs.PlateCarree() + fig, ax = plt.subplots( + 1, + 2, + subplot_kw={'projection': crs_pc}, + figsize=(12, 12), + layout='constrained', + ) + + ax[0].imshow(img, extent=full_extent, origin='upper', transform=crs_pc) + ax[1].imshow(label_data, extent=full_extent, origin='upper', transform=crs_pc) + + if save_to_file: + plt.savefig( + save_to_file, + dpi=300, + bbox_inches='tight', + ) + + if not quite: + plt.show() + + plt.close(fig) + + +def view_chips( + chips: models.ChipStack, + modality: models.Modality, + rgb_bands: tuple[int, int, int], + save_to_file: Path | None = None, + quite: bool = False, +): + merged_data = _merge_chips([chip.data for chip in chips], Path.cwd() / 'merged_data.tif') + merged_label = _merge_chips([chip.label for chip in chips], Path.cwd() / 'merged_label.tif') + try: + with rasterio.open(merged_data) as ds: + bounds = ds.bounds + full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] + band_data = ds.read() + + with rasterio.open(merged_label) as ds: + label_data = ds.read().squeeze() + + img = get_img(band_data, modality, rgb_bands) + + crs_pc = ccrs.PlateCarree() + # plot BANDS and geom + fig, ax = plt.subplots( + 1, + 2, + subplot_kw={'projection': crs_pc}, + figsize=(12, 12), + layout='constrained', + ) + + ax[0].imshow(img, extent=full_extent, origin='upper', transform=crs_pc) + ax[1].imshow(label_data, extent=full_extent, origin='upper', transform=crs_pc) + + if save_to_file: + plt.savefig( + save_to_file, + dpi=300, + bbox_inches='tight', + ) + + if not quite: + plt.show() + + plt.close(fig) + finally: + merged_data.unlink(missing_ok=True) + merged_label.unlink(missing_ok=True) + + +def _merge_chips( + chips: list[Path], + output_file: Path, +) -> Path: + datasets = [rasterio.open(p) for p in chips] + + try: + mosaic, transform = merge(datasets) + + out_meta = datasets[0].meta.copy() + out_meta.update( + { + 'height': mosaic.shape[1], + 'width': mosaic.shape[2], + 'transform': transform, + } + ) + + with rasterio.open(output_file, 'w', **out_meta) as dst: + if len(mosaic.shape) == 2: + dst.write(mosaic, 1) + else: + dst.write(mosaic) + finally: + for ds in datasets: + ds.close() + + return output_file + + +def get_img(band_data: np.ndarray, modality: models.Modality, rgb_bands: tuple[int, int, int]): + if 'HLS' in modality['id']: + img = get_hls_img(band_data, rgb_bands) + elif 'RTC' in modality['id']: + img = get_rtc_img(band_data, rgb_bands) + + return img + + +def get_hls_img(hls_data: np.ndarray, rgb_bands: tuple[int, int, int]) -> np.ndarray: + r_band, g_band, b_band = rgb_bands + + r = bytescale(np.sqrt(np.clip(hls_data[r_band] / 10000.0, 0, 2)), 0, 0.5) + g = bytescale(np.sqrt(np.clip(hls_data[g_band] / 10000.0, 0, 2)), 0, 0.5) + b = bytescale(np.sqrt(np.clip(hls_data[b_band] / 10000.0, 0, 2)), 0, 0.5) + + rgb = np.dstack((r, g, b)) + return rgb + + +def bytescale(arr, cmin=0, cmax=1, low=0, high=255): + # clip the data to be in the range of cmin to cmax + arr = np.clip(arr, cmin, cmax) + high = float(high) + low = float(low) + cmax = float(cmax) + cmin = float(cmin) + m = (high - low) / (cmax - cmin) # slope + b = high - (m * cmax) # intercept + arr = np.uint8((m * arr) + b) + return arr + + +def get_rtc_img(rtc_data: np.ndarray, rgb_bands: tuple[int, int, int]) -> np.ndarray: + r_band, g_band, b_band = rgb_bands + + r = normalize_image_array(np.sqrt(rtc_data[r_band]), 0.14, 0.52) + g = normalize_image_array(np.sqrt(rtc_data[g_band]), 0.05, 0.259) + b = normalize_image_array(np.sqrt(rtc_data[b_band]), 0.14, 0.52) + + img = np.stack([r, g, b], axis=-1) + + return img + + +def normalize_image_array(input_array: np.ndarray, vmin: float, vmax: float) -> np.ndarray: + input_array = input_array.astype(float) + scaled_array = (input_array - vmin) / (vmax - vmin) + scaled_array[np.isnan(input_array)] = 0 + normalized_array = np.round(np.clip(scaled_array, 0, 1) * 255).astype(np.uint8) + + return normalized_array diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index 7924506..ea8a780 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -45,6 +45,17 @@ def test_stack_bands(s2_local_files, s2_event, tmp_path): assert num_bands == len(bands) +def test_reproject_files(s2_local_files, s2_event, tmp_path): + reprojected_files = merge_modality.reproject_files(s2_local_files, tmp_path / 'wgs84') + + assert len(reprojected_files) == len(s2_local_files) + + for f in reprojected_files: + with rasterio.open(f) as ds: + epsg_code = ds.profile['crs'].to_epsg() + assert epsg_code == 4326 + + def test_warp_to_reference(s2_local_files, s2_event, tmp_path): merged = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') reprojected = merge_modality.reproject_files(merged, tmp_path / 'wgs84') @@ -66,14 +77,3 @@ def test_warp_to_reference(s2_local_files, s2_event, tmp_path): assert len(outputs) == 3 assert len(shapes) == 1 - - -def test_reproject_files(s2_local_files, s2_event, tmp_path): - reprojected_files = merge_modality.reproject_files(s2_local_files, tmp_path / 'wgs84') - - assert len(reprojected_files) == len(s2_local_files) - - for f in reprojected_files: - with rasterio.open(f) as ds: - epsg_code = ds.profile['crs'].to_epsg() - assert epsg_code == 4326 diff --git a/tests/test_view.py b/tests/test_view.py new file mode 100644 index 0000000..beec136 --- /dev/null +++ b/tests/test_view.py @@ -0,0 +1,41 @@ +from satchip import view, merge_modality, chip_data, models + + +def test_view_merged(s2_local_files, s2_event, tmp_path): + bands = tuple(b for b in models.HLS_S30['bands'] if b.id != 'Fmask') + + merged_bands = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged', bands) + stacked_data = merge_modality.stack_bands(merged_bands, stacked_filename=tmp_path / 'stacked.tif') + reprojected = merge_modality.reproject_files([stacked_data], tmp_path / 'reproj')[0] + + view.view_merged( + reprojected, s2_event, models.HLS_S30, rgb_bands=[2, 1, 0], save_to_file=tmp_path / 'output.png', quite=False + ) + + +def test_view_chip(warped_event_files, tmp_path): + fmask, label, data = warped_event_files + grid = chip_data.make_grid_from_reference(fmask) + + fmask_chips = chip_data.chip_data(grid, fmask, tmp_path / 'chips') + label_chips = chip_data.chip_data(grid, label, tmp_path / 'chips') + data_chips = chip_data.chip_data(grid, data, tmp_path / 'chips') + + stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30) + filtered = chip_data.filter_chips(stacks) + + view.view_chip(filtered[0], models.HLS_S30, [2, 1, 0]) + + +def test_view_chips(warped_event_files, tmp_path): + fmask, label, data = warped_event_files + grid = chip_data.make_grid_from_reference(fmask) + + fmask_chips = chip_data.chip_data(grid, fmask, tmp_path / 'chips') + label_chips = chip_data.chip_data(grid, label, tmp_path / 'chips') + data_chips = chip_data.chip_data(grid, data, tmp_path / 'chips') + + stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30) + filtered = chip_data.filter_chips(stacks) + + view.view_chips(filtered, models.HLS_S30, [2, 1, 0]) From 9e96f5f7a30990d53a0ebd76ce86138c8ade89b0 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 10 Apr 2026 10:43:08 -0800 Subject: [PATCH 19/21] Update notebook to be more usable --- .gitignore | 2 + notebooks/hwds-example.ipynb | 2018 ++++++++++++++++++++++++++++++--- src/satchip/merge_modality.py | 11 +- src/satchip/view.py | 5 + tests/test_merge_modality.py | 24 +- 5 files changed, 1870 insertions(+), 190 deletions(-) diff --git a/.gitignore b/.gitignore index 00087c9..718df2f 100644 --- a/.gitignore +++ b/.gitignore @@ -258,3 +258,5 @@ chips *.zip *.qgz hwds + +notebooks/data diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index d51e4d1..787cf4c 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -2,358 +2,2016 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "4dcd6420", "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import shutil\n", + "\n", + "import geopandas as gpd\n", + "import pandas as pd\n", + "\n", + "from satchip import models, download_data, merge_modality, generate_labels, chip_data, view" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2b0886a1", + "metadata": {}, + "outputs": [], + "source": [ + "shp_path = 'hwds_pristine'\n", + "df = gpd.read_file(shp_path)\n", + "\n", + "df['SwathDate'] = pd.to_datetime(df['SwathDate'], format='%Y-%m-%d')\n", + "df['HLSDate'] = pd.to_datetime(df['HLSDate'], format='%Y-%m-%d')\n", + "\n", + "#df = df[df['Visibility'] == '1']" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f1a32464", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'HLS_S30': {'modality': {'id': 'HLS_S30',\n", + " 'collection': 'HLSS30',\n", + " 'bands': (Band(id='B02', name='Blue', shortname='B'),\n", + " Band(id='B03', name='Green', shortname='G'),\n", + " Band(id='B04', name='Red', shortname='R'),\n", + " Band(id='B8A', name='NIR Narrow', shortname='N'),\n", + " Band(id='B11', name='SWIR 1', shortname='SW1'),\n", + " Band(id='B12', name='SWIR 2', shortname='SW2'),\n", + " Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))},\n", + " 'raw': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/raw'),\n", + " 'merged': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged'),\n", + " 'wgs84': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84'),\n", + " 'stacked': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/stacked'),\n", + " 'warped': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped'),\n", + " 'chips': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips'),\n", + " 'plots': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/plots')},\n", + " 'HLS_L30': {'modality': {'id': 'HLS_L30',\n", + " 'collection': 'HLSL30',\n", + " 'bands': (Band(id='B02', name='Blue', shortname='B'),\n", + " Band(id='B03', name='Green', shortname='G'),\n", + " Band(id='B04', name='Red', shortname='R'),\n", + " Band(id='B05', name='NIR Narrow', shortname='N'),\n", + " Band(id='B06', name='SWIR 1', shortname='SW1'),\n", + " Band(id='B07', name='SWIR 2', shortname='SW2'),\n", + " Band(id='Fmask', name='Cloud Mask', shortname='fmask'))},\n", + " 'raw': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/raw'),\n", + " 'merged': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged'),\n", + " 'wgs84': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84'),\n", + " 'stacked': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/stacked'),\n", + " 'warped': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/warped'),\n", + " 'chips': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/chips'),\n", + " 'plots': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/plots')}}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "DATA_PATH = Path.cwd() / 'data'\n", + "\n", + "def make_mod_paths(modality):\n", + " base_path = DATA_PATH / modality['id']\n", + "\n", + " return {\n", + " 'modality': modality,\n", + " 'raw': base_path / 'raw',\n", + " 'merged': base_path / 'merged',\n", + " 'wgs84': base_path / 'wgs84',\n", + " 'stacked': base_path / 'stacked',\n", + " 'warped': base_path / 'warped',\n", + " 'chips': base_path / 'chips',\n", + " 'plots': base_path / 'plots'\n", + " }\n", + "\n", + "mod_paths = {\n", + " modality['id']: make_mod_paths(modality) for modality in (models.HLS_S30, models.HLS_L30)\n", + "}\n", + "mod_paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d0202e9", + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/raw\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11050.57it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 280659.75it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 397355.12it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 95a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/95a.MASK.tif\n", + "Adding event files for 95a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 11599.83it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 461758.24it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 766471.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 102a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/102a.MASK.tif\n", + "Adding event files for 102a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 31564.69it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 637109.47it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 108/108 [00:00<00:00, 884736.00it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 106a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/106a.MASK.tif\n", + "Adding event files for 106a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12146.65it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 450731.18it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 786432.00it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 110a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110a.MASK.tif\n", + "Adding event files for 110a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11597.15it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 335544.32it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 496693.89it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 110c\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110c.MASK.tif\n", + "Adding event files for 110c\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10169.38it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 337042.29it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 490243.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 106d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/106d.MASK.tif\n", + "Adding event files for 106d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 9952.21it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 242757.14it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 408094.44it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 110g\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110g.MASK.tif\n", + "Adding event files for 110g\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13540.93it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 355282.22it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 820624.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 110e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110e.MASK.tif\n", + "Adding event files for 110e\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10591.68it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 267721.53it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 298408.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 126a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/126a.MASK.tif\n", + "Adding event files for 126a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13703.14it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 499983.26it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 477832.10it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 126b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/126b.MASK.tif\n", + "Adding event files for 126b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 11725.94it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 410312.35it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 665175.96it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 129a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/129a.MASK.tif\n", + "Adding event files for 129a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 23899.17it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 715615.85it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 917902.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 130a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/130a.MASK.tif\n", + "Adding event files for 130a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 22414.45it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 713924.09it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 829642.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 132a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132a.MASK.tif\n", + "Adding event files for 132a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12601.81it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 473338.38it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 356962.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 132b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132b.MASK.tif\n", + "Adding event files for 132b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14285.24it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 503316.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 766471.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 132c\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132c.MASK.tif\n", + "Adding event files for 132c\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13193.09it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 438938.79it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 751218.63it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 133a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/133a.MASK.tif\n", + "Adding event files for 133a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13699.41it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 488656.78it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 743817.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 134a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/134a.MASK.tif\n", + "Adding event files for 134a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 26786.40it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 720739.59it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 971028.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 134e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/134e.MASK.tif\n", + "Adding event files for 134e\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 15107.05it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 518882.97it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 909608.10it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 597b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/597b.MASK.tif\n", + "Adding event files for 597b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24889.96it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 634432.54it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1184274.07it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 611a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/611a.MASK.tif\n", + "Adding event files for 611a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13246.33it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 420598.73it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 613b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/613b.MASK.tif\n", + "Adding event files for 613b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13540.93it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 376546.00it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 816188.89it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 623d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/623d.MASK.tif\n", + "Adding event files for 623d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 20809.67it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 580749.78it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1006632.96it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 623a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/623a.MASK.tif\n", + "Adding event files for 623a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12841.89it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 496693.89it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 754974.72it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 889a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889a.MASK.tif\n", + "Adding event files for 889a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12384.76it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 522473.85it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 816188.89it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 889b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889b.MASK.tif\n", + "Adding event files for 889b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13572.58it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 520672.22it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 892a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/892a.MASK.tif\n", + "Adding event files for 892a\n", + "no HLS_S30 data for 915a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 49201.83it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 749148.01it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 162/162 [00:00<00:00, 1189977.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1079a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1079a.MASK.tif\n", + "Adding event files for 1079a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12855.01it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 308152.95it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 848286.20it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1069a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1069a.MASK.tif\n", + "Adding event files for 1069a\n", + "no HLS_S30 data for 1378a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11163.31it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 235194.62it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 510118.05it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1380a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1380a.MASK.tif\n", + "Adding event files for 1380a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24995.02it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 601573.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1016800.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 628a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/628a.MASK.tif\n", + "Adding event files for 628a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 27135.89it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 646382.47it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1037053.19it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 638a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/638a.MASK.tif\n", + "Adding event files for 638a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 27245.57it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 681692.75it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1090216.20it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 116d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116d.MASK.tif\n", + "Adding event files for 116d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14768.68it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 499983.26it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 441505.68it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 116a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116a.MASK.tif\n", + "Adding event files for 116a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12733.59it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 452080.67it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 736560.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 116e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116e.MASK.tif\n", + "Adding event files for 116e\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10817.81it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 344737.32it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 487080.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 648a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/648a.MASK.tif\n", + "Adding event files for 648a\n", + "no HLS_S30 data for 1052a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 34015.53it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 793318.44it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 108/108 [00:00<00:00, 1402429.82it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1055a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1055a.MASK.tif\n", + "Adding event files for 1055a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24254.27it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 633102.49it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1070886.13it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1056a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1056a.MASK.tif\n", + "Adding event files for 1056a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 27889.72it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 456178.08it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1110256.94it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1056c\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1056c.MASK.tif\n", + "Adding event files for 1056c\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13274.28it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 412554.49it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 867787.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1347a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1347a.MASK.tif\n", + "Adding event files for 1347a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 23226.42it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 681692.75it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1139584.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1064a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1064a.MASK.tif\n", + "Adding event files for 1064a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13982.31it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 503316.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 898779.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1338a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1338a.MASK.tif\n", + "Adding event files for 1338a\n", + "no HLS_S30 data for 1344a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11209.72it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 204047.22it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 414821.27it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1373a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373a.MASK.tif\n", + "Adding event files for 1373a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10433.59it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 324023.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 513588.24it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1373d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373d.MASK.tif\n", + "Adding event files for 1373d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13342.31it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 482411.96it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 662258.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_S30 data for 1373e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373e.MASK.tif\n", + "Adding event files for 1373e\n", + "no HLS_S30 data for 1376a\n", + "{'id': 'HLS_L30', 'collection': 'HLSL30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B05', name='NIR Narrow', shortname='N'), Band(id='B06', name='SWIR 1', shortname='SW1'), Band(id='B07', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/raw\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10092.17it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 261056.27it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 331129.26it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 95a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/95a.MASK.tif\n", + "Adding event files for 95a\n", + "no HLS_L30 data for 102a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 19294.01it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 737280.00it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1301680.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 106a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/106a.MASK.tif\n", + "Adding event files for 106a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12599.29it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 441505.68it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 753467.78it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 110a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110a.MASK.tif\n", + "Adding event files for 110a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 9554.22it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 288598.90it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 439961.96it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 110c\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110c.MASK.tif\n", + "Adding event files for 110c\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10265.06it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 186690.09it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 433893.52it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 106d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/106d.MASK.tif\n", + "Adding event files for 106d\n", + "no HLS_L30 data for 110g\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13089.47it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 443060.28it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 110e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110e.MASK.tif\n", + "Adding event files for 110e\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10765.67it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 268865.64it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 301026.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 126a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/126a.MASK.tif\n", + "Adding event files for 126a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13122.24it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 441505.68it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 776722.96it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 126b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/126b.MASK.tif\n", + "Adding event files for 126b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12715.15it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 446202.55it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 744551.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 129a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/129a.MASK.tif\n", + "Adding event files for 129a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 16514.09it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 699050.67it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 975419.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 130a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/130a.MASK.tif\n", + "Adding event files for 130a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 21204.77it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 621378.37it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1027176.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 132a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132a.MASK.tif\n", + "Adding event files for 132a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13877.70it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 432402.47it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 467766.25it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 132b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132b.MASK.tif\n", + "Adding event files for 132b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 10378.52it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 408536.10it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 577197.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 132c\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132c.MASK.tif\n", + "Adding event files for 132c\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 14810.40it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 454256.75it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 133a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/133a.MASK.tif\n", + "Adding event files for 133a\n", + "no HLS_L30 data for 134a\n", + "no HLS_L30 data for 134e\n", + "no HLS_L30 data for 597b\n", + "no HLS_L30 data for 611a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12875.18it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 435394.88it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 801459.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 613b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/613b.MASK.tif\n", + "Adding event files for 613b\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11924.67it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 427990.20it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 629145.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 623d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623d.MASK.tif\n", + "Adding event files for 623d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 20947.08it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 640351.76it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 967916.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 623a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623a.MASK.tif\n", + "Adding event files for 623a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11818.27it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 449389.71it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 758006.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 889a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/889a.MASK.tif\n", + "Adding event files for 889a\n", + "no HLS_L30 data for 889b\n", + "no HLS_L30 data for 892a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13824.34it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 435394.88it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 571950.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 915a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/915a.MASK.tif\n", + "Adding event files for 915a\n", + "no HLS_L30 data for 1079a\n", + "no HLS_L30 data for 1069a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 10954.05it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 462607.06it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1378a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1378a.MASK.tif\n", + "Adding event files for 1378a\n", + "no HLS_L30 data for 1380a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 21445.10it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 626015.52it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 939023.28it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 628a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/628a.MASK.tif\n", + "Adding event files for 628a\n", + "no HLS_L30 data for 638a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 20717.73it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 624462.13it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1031386.23it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 116d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116d.MASK.tif\n", + "Adding event files for 116d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12356.78it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 354448.23it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 658791.20it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 116a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116a.MASK.tif\n", + "Adding event files for 116a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12492.96it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 422245.37it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 635500.61it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 116e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116e.MASK.tif\n", + "Adding event files for 116e\n", + "no HLS_L30 data for 648a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12104.77it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 375609.31it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 714938.18it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1052a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1052a.MASK.tif\n", + "Adding event files for 1052a\n", + "no HLS_L30 data for 1055a\n", + "no HLS_L30 data for 1056a\n", + "no HLS_L30 data for 1056c\n", + "no HLS_L30 data for 1347a\n", + "no HLS_L30 data for 1064a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11621.79it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 446202.55it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 727335.95it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1338a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1338a.MASK.tif\n", + "Adding event files for 1338a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12651.23it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 460912.53it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 436906.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1344a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1344a.MASK.tif\n", + "Adding event files for 1344a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 9725.55it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 272357.40it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 343795.41it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1373a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373a.MASK.tif\n", + "Adding event files for 1373a\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 11376.95it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 283398.92it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 413911.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1373d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373d.MASK.tif\n", + "Adding event files for 1373d\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12795.31it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 455902.61it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 619847.88it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1373e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373e.MASK.tif\n", + "Adding event files for 1373e\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/wbhorn/miniforge3/envs/satchip/lib/python3.14/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10602.39it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 266587.12it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 473041.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found HLS_L30 data for 1376a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1376a.MASK.tif\n", + "Adding event files for 1376a\n" ] } ], "source": [ - "from pathlib import Path\n", + "event_files = []\n", "\n", - "import geopandas as gpd\n", - "import pandas as pd\n", + "for paths in mod_paths.values():\n", + " print(paths['modality'], paths['raw'])\n", "\n", - "from satchip import models, download_data, merge_modality, generate_labels, chip_data, view" + " for idx, row in df.iterrows():\n", + " modality = paths['modality']\n", + "\n", + " damage_event = models.Event(\n", + " name=row['HLSID'], \n", + " date=row['HLSDate'], \n", + " wgs84_geometry=row['geometry'], \n", + " buffer_m=10000\n", + " )\n", + "\n", + " local = download_data.download_data(damage_event, modality, paths['raw'])\n", + "\n", + " if not local:\n", + " print(f'no {modality[\"id\"]} data for {damage_event.name}')\n", + " continue\n", + " else:\n", + " print(f'Found {modality[\"id\"]} data for {damage_event.name}')\n", + "\n", + " merged_event = merge_modality.merge_modality(\n", + " local, \n", + " modality, \n", + " event=damage_event, \n", + " output_path=paths['merged']\n", + " )\n", + "\n", + " stacked_filename = paths['stacked'] / f'{damage_event.name}.{modality[\"id\"]}.stacked.tif'\n", + " data_bands, fmask = merged_event[:-1], merged_event[-1]\n", + "\n", + " stacked = merge_modality.stack_bands(data_bands, stacked_filename)\n", + "\n", + " label = generate_labels.binary_mask_from_template(stacked, damage_event, paths['wgs84'])\n", + " mask, bands, fmask = merge_modality.warp_to_reference(\n", + " reference_path=label,\n", + " data_files=[stacked, fmask],\n", + " output_dir=paths['warped'],\n", + " bounding_box_wgs84=damage_event.buffered_geometry().bounds,\n", + " )\n", + " print(f'Adding event files for {damage_event.name}')\n", + " event_files.append((damage_event, modality, (mask, bands, fmask)))\n", + "\n", + " view.view_merged(\n", + " bands, \n", + " damage_event, \n", + " modality, \n", + " rgb_bands=[2, 1, 0],\n", + " quite=True, \n", + " save_to_file=paths['plots'] / f'{damage_event.name}.{modality[\"id\"]}.merged.plot.png'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "a5c47f55", + "metadata": {}, + "source": [ + "Remove any bad aquisitions from `data/{modality}/plots` by remove the .png " ] }, { "cell_type": "code", - "execution_count": null, - "id": "257c1816", + "execution_count": 43, + "id": "ed4d4521", "metadata": {}, "outputs": [], "source": [ - "modality = models.HLS_S30\n", + "def backup_plots():\n", + " for paths in mod_paths.values():\n", + " plots = paths['plots']\n", + " plots_backup = plots.parent / 'plots-backup'\n", + " shutil.copytree(plots, plots_backup, dirs_exist_ok=True)\n", "\n", - "DATA_PATH = Path.cwd() / 'data'\n", - "MODALITY_PATH = DATA_PATH / modality['id']\n", - "\n", - "RAW_DATA_PATH = MODALITY_PATH / 'raw'\n", - "MERGED_PATH = MODALITY_PATH / 'merged'\n", - "REPROJECTED_PATH = MODALITY_PATH / 'wgs84'\n", - "STACKED_PATH = MODALITY_PATH / 'stacked'\n", - "WARPED_PATH = MODALITY_PATH / 'warped'\n", - "CHIPS_PATH = MODALITY_PATH / 'chips'" + "backup_plots()" ] }, { "cell_type": "code", "execution_count": null, - "id": "2b0886a1", + "id": "519b634b", "metadata": {}, "outputs": [], "source": [ - "shp_path = 'hwds_pristine'\n", - "df = gpd.read_file(shp_path)\n", + "def restore_plots():\n", + " for paths in mod_paths.values():\n", + " plots = paths['plots']\n", + " plots_backup = plots.parent / 'plots-backup'\n", + " shutil.copytree(plots_backup, plots, dirs_exist_ok=True)\n", "\n", - "df['SwathDate'] = pd.to_datetime(df['SwathDate'], format='%Y-%m-%d')\n", - "df['HLSDate'] = pd.to_datetime(df['HLSDate'], format='%Y-%m-%d')" + "#restore_plots()" ] }, { "cell_type": "code", - "execution_count": null, - "id": "3dc85728", + "execution_count": 44, + "id": "44902869", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "SwathDate 2019-07-09 00:00:00\n", - "HLSID 102a\n", - "MODISID 102\n", - "Visibility 2\n", - "HLSDate 2019-07-17 00:00:00\n", - "geometry POLYGON Z ((-99.49996223199997 40.270482574000...\n", - "Name: 1, dtype: object" + "{'HLS_S30': ['628a',\n", + " '1079a',\n", + " '597b',\n", + " '1069a',\n", + " '1380a',\n", + " '116d',\n", + " '613b',\n", + " '648a',\n", + " '110e',\n", + " '110g',\n", + " '129a',\n", + " '126b',\n", + " '638a',\n", + " '116e',\n", + " '1338a',\n", + " '1347a',\n", + " '102a',\n", + " '106d',\n", + " '611a',\n", + " '1373e',\n", + " '623d',\n", + " '892a',\n", + " '1373d',\n", + " '110a',\n", + " '1064a',\n", + " '1056a',\n", + " '110c',\n", + " '889b',\n", + " '106a',\n", + " '132b',\n", + " '134e',\n", + " '1055a',\n", + " '889a',\n", + " '95a',\n", + " '133a',\n", + " '126a',\n", + " '134a',\n", + " '132c',\n", + " '1056c',\n", + " '130a',\n", + " '623a',\n", + " '1373a',\n", + " '116a',\n", + " '132a'],\n", + " 'HLS_L30': ['132a',\n", + " '116a',\n", + " '106d',\n", + " '95a',\n", + " '126b',\n", + " '1344a',\n", + " '1373d',\n", + " '110e',\n", + " '1378a',\n", + " '1338a',\n", + " '110c',\n", + " '132b',\n", + " '1376a',\n", + " '133a',\n", + " '613b',\n", + " '915a',\n", + " '132c',\n", + " '1052a',\n", + " '116d',\n", + " '623d',\n", + " '1373e',\n", + " '628a',\n", + " '889a',\n", + " '1373a',\n", + " '130a',\n", + " '126a',\n", + " '106a',\n", + " '110a',\n", + " '623a',\n", + " '129a',\n", + " '116e']}" ] }, - "execution_count": 4, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "swath = df.iloc[1]\n", - "swath" + "keepers = {}\n", + "\n", + "for paths in mod_paths.values():\n", + " modality = paths['modality']\n", + " mod_keepers = [plot.name.split('.')[0] for plot in paths['plots'].glob('*.png')]\n", + " keepers[modality['id']] = mod_keepers\n", + "\n", + "keepers" ] }, { "cell_type": "code", - "execution_count": null, - "id": "6a611425", + "execution_count": 33, + "id": "6f59f9a1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Event(name='102a', date=Timestamp('2019-07-09 00:00:00'), wgs84_geometry=, buffer_m=10000)\n" + "(Event(name='95a', date=Timestamp('2018-07-02 00:00:00'), wgs84_geometry=, buffer_m=10000), {'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}, (PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.MASK.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.HLS_S30.stacked.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.HLS_S30.2018-07-02.Fmask.tif')))\n" ] } ], "source": [ - "event = models.Event(name=swath['HLSID'], date=swath['SwathDate'], wgs84_geometry=swath['geometry'], buffer_m=10000)\n", - "print(event)" + "print(event_files[0])\n", + "evt, modality, (m, b, f) = event_files[0]" ] }, { "cell_type": "code", - "execution_count": null, - "id": "0d0202e9", + "execution_count": 34, + "id": "19734cc8", + "metadata": {}, + "outputs": [], + "source": [ + "all_base = DATA_PATH / 'chips'\n", + "output_base = DATA_PATH / 'output'\n", + "\n", + "chip_paths = {\n", + " 'all': {\n", + " 'label': all_base / 'LABEL',\n", + " 'hls': all_base / 'HLS',\n", + " 'other': all_base / 'OTHER',\n", + " 'plots': all_base / 'PLOTS',\n", + " },\n", + " 'output': {\n", + " 'label': output_base / 'LABEL',\n", + " 'hls': output_base / 'HLS',\n", + " 'other': output_base / 'OTHER',\n", + " 'plots': all_base / 'PLOTS',\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "3c4cee14", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Logging in to earthaccess\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 9419.52it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 277564.24it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 354448.23it/s]\n" + "Chipping: 95a 95a.MASK.tif 95a.HLS_S30.stacked.tif 95a.HLS_S30.2018-07-02.Fmask.tif\n", + "Chipping: 126a 126a.MASK.tif 126a.HLS_S30.stacked.tif 126a.HLS_S30.2018-08-11.Fmask.tif\n", + "Chipping: 116a 116a.MASK.tif 116a.HLS_S30.stacked.tif 116a.HLS_S30.2020-06-17.Fmask.tif\n", + "Chipping: 116e 116e.MASK.tif 116e.HLS_S30.stacked.tif 116e.HLS_S30.2020-06-17.Fmask.tif\n", + "Chipping: 106a 106a.MASK.tif 106a.HLS_L30.stacked.tif 106a.HLS_L30.2019-08-28.fmask.tif\n", + "Chipping: 126a 126a.MASK.tif 126a.HLS_L30.stacked.tif 126a.HLS_L30.2018-08-11.fmask.tif\n", + "Already chipped event 126a: skipping...\n", + "Chipping: 116a 116a.MASK.tif 116a.HLS_L30.stacked.tif 116a.HLS_L30.2020-06-17.fmask.tif\n", + "Already chipped event 116a: skipping...\n" ] } ], "source": [ - "local_files = download_data.download_data(event, modality, RAW_DATA_PATH)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b99ff34", - "metadata": {}, - "outputs": [], - "source": [ - "merged_event = merge_modality.merge_modality(local_files, modality, event=event, output_path=MERGED_PATH)" + "chipped_events = set() \n", + "filtered_chips = []\n", + "all_chips = []\n", + "\n", + "for evt, modality, (m, b, f) in event_files:\n", + " if evt.name not in keepers[modality['id']]:\n", + " continue\n", + "\n", + " print('Chipping: ', evt.name, m.name, b.name, f.name)\n", + " if evt.name in chipped_events:\n", + " print(f'Already chipped event {evt.name}: skipping...')\n", + " continue\n", + "\n", + " chipped_events.add(evt.name)\n", + " grid = chip_data.make_grid_from_reference(m)\n", + "\n", + " label_chips = chip_data.chip_data(grid, m, chip_paths['all']['label'])\n", + " data_chips = chip_data.chip_data(grid, b, chip_paths['all']['hls'])\n", + " fmask_chips = chip_data.chip_data(grid, f, chip_paths['all']['other'])\n", + "\n", + " stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, modality)\n", + " all_chips += stacks\n", + " view.view_chips(\n", + " stacks, \n", + " modality, \n", + " rgb_bands=[2, 1, 0], \n", + " save_to_file=chip_paths['all']['plots'] / f'{evt.name}.chips.png', \n", + " quite=True\n", + " )\n", + " \n", + " filtered = chip_data.filter_chips(stacks)\n", + "\n", + " if len(filtered) == 0:\n", + " print(f'Filtered out all chips for {evt.name}')\n", + " continue\n", + "\n", + " filtered_chips += filtered\n", + " view.view_chips(\n", + " filtered, \n", + " modality, \n", + " rgb_bands=[2, 1, 0], \n", + " save_to_file=chip_paths['output']['plots'] / f'{evt.name}.filtered.png', \n", + " quite=True\n", + " )" ] }, { "cell_type": "code", - "execution_count": null, - "id": "e6ce6b53", + "execution_count": 47, + "id": "252dff47", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.B.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.G.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.R.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.N.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW1.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.SW2.tif'),\n", - " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/merged/102a.HLS_S30.2019-07-09.Fmask.tif')]" + "(24, 112)" ] }, - "execution_count": 8, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "merged_event" + "len(filtered), len(all_chips)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "9673d689", + "execution_count": 50, + "id": "6a7dca8f", "metadata": {}, "outputs": [], "source": [ - "stacked_filename = MERGED_PATH / f'{swath[\"HLSID\"]}.stacked.tif'\n", - "data_bands, fmask = merged_event[:-1], merged_event[-1]\n", + "chip_paths['output']['hls'].mkdir(exist_ok=True, parents=True)\n", + "chip_paths['output']['label'].mkdir(exist_ok=True, parents=True)\n", + "\n", + "for chip in filtered_chips:\n", + " output_data_path = chip_paths['output']['hls'] / chip.data.name \n", + " output_label_path = chip_paths['output']['label'] / chip.label.name\n", "\n", - "stacked = merge_modality.stack_bands(data_bands, stacked_filename)" + " shutil.copy2(chip.data, output_data_path)\n", + " shutil.copy2(chip.label, output_label_path)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "e6f7e30b", + "execution_count": 51, + "id": "b1c32b82", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "40" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "stacked, fmask = merge_modality.reproject_files([stacked, fmask], REPROJECTED_PATH)" + "len(list(chip_paths['output']['hls'].glob('*.tif')))\n", + "len(list(chip_paths['output']['label'].glob('*.tif')))" ] }, { "cell_type": "code", "execution_count": null, - "id": "9d9b6664", + "id": "6f82016d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/102a.MASK.tif\n" + "86 308\n" ] } ], "source": [ - "label = generate_labels.binary_mask_from_template(stacked, event, REPROJECTED_PATH)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3180399a", - "metadata": {}, - "outputs": [], - "source": [ - "mask, bands, fmask = merge_modality.warp_to_reference(\n", - " reference_path=label,\n", - " data_files=[stacked, fmask],\n", - " output_dir=WARPED_PATH,\n", - " bounding_box_wgs84=event.buffered_geometry().bounds,\n", - ")" + "#len(filtered_chips)\n", + "#filtered_chips[0]\n", + "#CHIPS_PLOTS = PLOT_PATH / 'CHIPS'\n", + "#print(len(filtered_chips), len(all_chips))\n", + "#for chip in filtered_chips:\n", + "# plot_name = f\"{chip.data.name.split('.stacked')[0]}.chip.png\"\n", + "# view.view_chip(chip, models.HLS_S30, [2, 1, 0], save_to_file=CHIPS_PLOTS / plot_name, quite=True)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "7e3e9a4b", + "execution_count": 52, + "id": "3049d797", "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "import rasterio\n", "\n", - "with rasterio.open(bands) as ds:\n", - " bounds = ds.bounds\n", - " full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top]\n", "\n", - " band_data = ds.read()" + "def calculate_stats(chip_stacks, n_bands) -> tuple:\n", + " mean = np.zeros(n_bands, dtype=np.float64)\n", + " M2 = np.zeros(n_bands, dtype=np.float64)\n", + " count = np.zeros(n_bands, dtype=np.float64)\n", + "\n", + " for chip in chip_stacks:\n", + " with rasterio.open(chip.data) as src:\n", + " band_data = src.read()\n", + " count, mean, M2 = 0, 0, 0\n", + "\n", + " _, H, W = band_data.shape\n", + "\n", + " batch_count = H * W\n", + " batch_mean = band_data.mean(axis=(1, 2))\n", + " batch_var = band_data.var(axis=(1, 2))\n", + "\n", + " delta = batch_mean - mean\n", + " total_count = count + batch_count\n", + "\n", + " mean = mean + delta * (batch_count / total_count)\n", + " M2 = (\n", + " M2\n", + " + batch_var * batch_count\n", + " + (delta**2) * count * batch_count / total_count\n", + " )\n", + " count = total_count\n", + "\n", + " variance = M2 / count\n", + " std = np.sqrt(variance)\n", + "\n", + " return mean, std\n" ] }, { "cell_type": "code", - "execution_count": null, - "id": "9513cb17", + "execution_count": 53, + "id": "00880cff", "metadata": {}, "outputs": [ - { - "ename": "ValueError", - "evalue": "Expected a projection subclass. Cannot handle a in imshow.", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m view.view_merged(bands, event, modality, [\u001b[32m2\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m0\u001b[39m], quite=\u001b[38;5;28;01mFalse\u001b[39;00m)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Repositories/fm/satchip/src/satchip/view.py:41\u001b[39m, in \u001b[36mview_merged\u001b[39m\u001b[34m(stacked_data_file, event, modality, rgb_bands, save_to_file, quite)\u001b[39m\n\u001b[32m 31\u001b[39m fig, ax = plt.subplots(\n\u001b[32m 32\u001b[39m \u001b[32m1\u001b[39m,\n\u001b[32m 33\u001b[39m \u001b[32m1\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 36\u001b[39m layout=\u001b[33m'\u001b[39m\u001b[33mconstrained\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m 37\u001b[39m )\n\u001b[32m 39\u001b[39m event_geom = event.wgs84_geometry\n\u001b[32m---> \u001b[39m\u001b[32m41\u001b[39m \u001b[43max\u001b[49m\u001b[43m.\u001b[49m\u001b[43mimshow\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextent\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfull_extent\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morigin\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mupper\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtransform\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 42\u001b[39m ax.add_geometries([event_geom], edgecolor=\u001b[33m'\u001b[39m\u001b[33mred\u001b[39m\u001b[33m'\u001b[39m, linewidth=\u001b[32m2\u001b[39m, facecolor=\u001b[33m'\u001b[39m\u001b[33mnone\u001b[39m\u001b[33m'\u001b[39m, crs=crs_pc)\n\u001b[32m 44\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m save_to_file:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/miniforge3/envs/satchip/lib/python3.14/site-packages/cartopy/mpl/geoaxes.py:291\u001b[39m, in \u001b[36m_add_transform..wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 286\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mInvalid transform: Spherical \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc.\u001b[34m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 287\u001b[39m \u001b[33m'\u001b[39m\u001b[33mis not supported - consider using \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 288\u001b[39m \u001b[33m'\u001b[39m\u001b[33mPlateCarree/RotatedPole.\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 290\u001b[39m kwargs[\u001b[33m'\u001b[39m\u001b[33mtransform\u001b[39m\u001b[33m'\u001b[39m] = transform\n\u001b[32m--> \u001b[39m\u001b[32m291\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/miniforge3/envs/satchip/lib/python3.14/site-packages/cartopy/mpl/geoaxes.py:1287\u001b[39m, in \u001b[36mGeoAxes.imshow\u001b[39m\u001b[34m(self, img, *args, **kwargs)\u001b[39m\n\u001b[32m 1284\u001b[39m kwargs[\u001b[33m'\u001b[39m\u001b[33morigin\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33m'\u001b[39m\u001b[33mlower\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1286\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(transform, ccrs.Projection):\n\u001b[32m-> \u001b[39m\u001b[32m1287\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m'\u001b[39m\u001b[33mExpected a projection subclass. Cannot \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1288\u001b[39m \u001b[33m'\u001b[39m\u001b[33mhandle a \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m in imshow.\u001b[39m\u001b[33m'\u001b[39m % \u001b[38;5;28mtype\u001b[39m(transform))\n\u001b[32m 1290\u001b[39m target_extent = \u001b[38;5;28mself\u001b[39m.get_extent(\u001b[38;5;28mself\u001b[39m.projection)\n\u001b[32m 1291\u001b[39m regrid_shape = kwargs.pop(\u001b[33m'\u001b[39m\u001b[33mregrid_shape\u001b[39m\u001b[33m'\u001b[39m, \u001b[32m750\u001b[39m)\n", - "\u001b[31mValueError\u001b[39m: Expected a projection subclass. Cannot handle a in imshow." - ] - }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABLsAAAJnCAYAAAB/KlGVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAEX9JREFUeJzt3LFq41AQQFFlCcKVazUq/P8f5cKNa1dGjYIWttysghU2XM6ph2Hqy+O9reu6DgAAAAAQ8Ot/HwAAAAAARxG7AAAAAMgQuwAAAADIELsAAAAAyBC7AAAAAMgQuwAAAADIELsAAAAAyHjfO/h8PodlWb73GgAAAAD4i3Ech9PpNLwcu7bQdblchvv9vmccAAAAAA43TdNwvV4/DV67Ytf2omsLXbfbbTifz0feCAAAAAD/9Hg8hnmef3eql2PXH1voErsAAAAA+Kl8UA8AAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAhtgFAAAAQIbYBQAAAECG2AUAAABAxvtXhh+Px/ddAgAAAAAvdqldsWscx2GapmGe511LAQAAAOBoW5/aOtVn3tZ1Xfcsez6fw7IsR90GAAAAAF+yha7T6XRM7AIAAACAn84H9QAAAABkiF0AAAAAZIhdAAAAAGSIXQAAAABkiF0AAAAAZIhdAAAAAGSIXQAAAAAMFR+6uDuDhX37uwAAAABJRU5ErkJggg==", "text/plain": [ - "
" + "[PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.B.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.G.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.R.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.N.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.SW1.tif'),\n", + " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.SW2.tif')]" ] }, + "execution_count": 53, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "view.view_merged(bands, event, modality, [2, 1, 0], quite=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f65b3ed", - "metadata": {}, - "outputs": [], - "source": [ - "grid = chip_data.make_grid_from_reference(mask)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c4cee14", - "metadata": {}, - "outputs": [], - "source": [ - "label_chips = chip_data.chip_data(grid, mask, CHIPS_PATH / 'LABELS')\n", - "data_chips = chip_data.chip_data(grid, bands, CHIPS_PATH / modality['id'])\n", - "fmask_chips = chip_data.chip_data(grid, fmask, CHIPS_PATH / 'FMASK')" + "data_bands" ] }, { "cell_type": "code", - "execution_count": null, - "id": "8d3ac758", + "execution_count": 54, + "id": "f68fa713", "metadata": {}, "outputs": [], "source": [ - "stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, models.HLS_S30)\n", - "filtered = chip_data.filter_chips(stacks)" + "chip_means, chip_stds = calculate_stats(filtered_chips, 6)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "e5c082c9", + "execution_count": 55, + "id": "7a6d8b65", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[ChipStack(id='001.001', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.001.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.001.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.001.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", - " ChipStack(id='001.002', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.002.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.002.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.002.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", - " ChipStack(id='001.003', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.003.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.003.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.003.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", - " ChipStack(id='001.004', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.004.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.004.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.004.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}),\n", - " ChipStack(id='001.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/HLS_S30/001.005.102a.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/FMASK/001.005.102a.HLS_S30.2019-07-09.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/chips/LABELS/001.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))})]" + "415" ] }, - "execution_count": 36, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "filtered" + "import json\n", + "\n", + "\n", + "bands = models.HLS_S30['bands'][:-1]\n", + "bands\n", + "\n", + "stats_file = {\n", + " 'HLS':{\n", + " 'means': {},\n", + " 'stds': {},\n", + " }\n", + "}\n", + "\n", + "for mean, std, band in zip(chip_means, chip_stds, bands):\n", + " stats_file['HLS']['means'][band.shortname] = float(mean)\n", + " stats_file['HLS']['stds'][band.shortname] = float(std)\n", + "\n", + "stats_file\n", + "\n", + "(output_base / 'statistics.json').write_text(json.dumps(stats_file, indent=2))" ] }, { "cell_type": "code", "execution_count": null, - "id": "6b5369e9", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "549d0513", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "70358b1c", + "id": "2aed8fc1", "metadata": {}, "outputs": [], "source": [] diff --git a/src/satchip/merge_modality.py b/src/satchip/merge_modality.py index 4dd4b0e..cc3679a 100644 --- a/src/satchip/merge_modality.py +++ b/src/satchip/merge_modality.py @@ -19,7 +19,10 @@ def merge_modality( output_path: Path, selected_bands: list[models.Band] | None = None, ) -> list[Path]: + reproj_path = output_path / 'wgs84' + output_path.mkdir(exist_ok=True, parents=True) + reproj_path.mkdir(exist_ok=True, parents=True) if len(modality_files) == 0: print(f'Warning: no data for {event.name}') @@ -35,7 +38,8 @@ def merge_modality( merged_name = _make_merge_name(event.name, event.date, band.shortname, modality['id']) - merged_band_path = merge_files(band_files, output_file=output_path / merged_name) + reprojected_files = reproject_files(band_files, reproj_path) + merged_band_path = merge_files(reprojected_files, output_file=output_path / merged_name) merged.append(merged_band_path) @@ -51,11 +55,6 @@ def _make_merge_name(event_name: str, start_date: datetime.datetime, band: str, def merge_files(files: list[Path], output_file: Path) -> Path: band_datasets = [rasterio.open(band_file) for band_file in files] - reference_crs = band_datasets[0].crs - for ds in band_datasets[1:]: - if ds.crs != reference_crs: - ds.crs = reference_crs - try: mosaic, out_trans = merge(band_datasets) mosaic = np.squeeze(mosaic) diff --git a/src/satchip/view.py b/src/satchip/view.py index 83d7378..0de8d2f 100644 --- a/src/satchip/view.py +++ b/src/satchip/view.py @@ -18,6 +18,7 @@ def view_merged( save_to_file: Path | None = None, quite: bool = False, ): + save_to_file.parent.mkdir(exist_ok=True, parents=True) crs_pc = ccrs.PlateCarree() with rasterio.open(stacked_data_file) as ds: @@ -61,6 +62,8 @@ def view_chip( save_to_file: Path | None = None, quite: bool = False, ): + save_to_file.parent.mkdir(exist_ok=True, parents=True) + with rasterio.open(chip.data) as ds: bounds = ds.bounds full_extent = [bounds.left, bounds.right, bounds.bottom, bounds.top] @@ -103,6 +106,8 @@ def view_chips( save_to_file: Path | None = None, quite: bool = False, ): + save_to_file.parent.mkdir(exist_ok=True, parents=True) + merged_data = _merge_chips([chip.data for chip in chips], Path.cwd() / 'merged_data.tif') merged_label = _merge_chips([chip.label for chip in chips], Path.cwd() / 'merged_label.tif') try: diff --git a/tests/test_merge_modality.py b/tests/test_merge_modality.py index ea8a780..68b0ac5 100644 --- a/tests/test_merge_modality.py +++ b/tests/test_merge_modality.py @@ -1,6 +1,22 @@ +from pathlib import Path import rasterio -from satchip import generate_labels, merge_modality, models +from satchip import generate_labels, merge_modality, models, download_data + + +def test_multi_projection_event(pristine_gdf): + swath = pristine_gdf.iloc[2] + + data_path = Path(__file__).parent / 'data' + raw_path = data_path / 'raw' + merge_path = Path(__file__).parent / 'data' / 'merge' + + modality = models.HLS_S30 + + event = models.Event(name=swath['HLSID'], date=swath['SwathDate'], wgs84_geometry=swath['geometry'], buffer_m=10000) + + local_files = download_data.download_data(event, modality, raw_path) + merged_event = merge_modality.merge_modality(local_files, modality, event=event, output_path=merge_path) def test_make_merge_name(s2_event): @@ -56,7 +72,7 @@ def test_reproject_files(s2_local_files, s2_event, tmp_path): assert epsg_code == 4326 -def test_warp_to_reference(s2_local_files, s2_event, tmp_path): +def test_align_to_reference(s2_local_files, s2_event, tmp_path): merged = merge_modality.merge_modality(s2_local_files, models.HLS_S30, s2_event, tmp_path / 'merged') reprojected = merge_modality.reproject_files(merged, tmp_path / 'wgs84') @@ -66,8 +82,8 @@ def test_warp_to_reference(s2_local_files, s2_event, tmp_path): stacked = merge_modality.stack_bands(bands, stacked_filename=tmp_path / 'stacked.tif') label = generate_labels.binary_mask_from_template(stacked, s2_event, tmp_path) - outputs = merge_modality.warp_to_reference( - label, [stacked, fmask], tmp_path / 'warped', s2_event.buffered_geometry().bounds + outputs = merge_modality.align_to_reference( + label, [stacked, fmask], tmp_path / 'aligned', s2_event.buffered_geometry().bounds ) shapes = set() From bff8f23443b232540ce907e40afd2692de9127ea Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 20 Apr 2026 14:50:43 -0800 Subject: [PATCH 20/21] Work on chipping notebook --- notebooks/hwds-example.ipynb | 133 ++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 57 deletions(-) diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index 787cf4c..3c08ae4 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -1557,7 +1557,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "ed4d4521", "metadata": {}, "outputs": [], @@ -1568,12 +1568,12 @@ " plots_backup = plots.parent / 'plots-backup'\n", " shutil.copytree(plots, plots_backup, dirs_exist_ok=True)\n", "\n", - "backup_plots()" + "# backup_plots()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "id": "519b634b", "metadata": {}, "outputs": [], @@ -1584,12 +1584,12 @@ " plots_backup = plots.parent / 'plots-backup'\n", " shutil.copytree(plots_backup, plots, dirs_exist_ok=True)\n", "\n", - "#restore_plots()" + "restore_plots()" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 57, "id": "44902869", "metadata": {}, "outputs": [ @@ -1602,10 +1602,7 @@ " '1069a',\n", " '1380a',\n", " '116d',\n", - " '613b',\n", " '648a',\n", - " '110e',\n", - " '110g',\n", " '129a',\n", " '126b',\n", " '638a',\n", @@ -1614,66 +1611,44 @@ " '1347a',\n", " '102a',\n", " '106d',\n", - " '611a',\n", " '1373e',\n", " '623d',\n", " '892a',\n", " '1373d',\n", - " '110a',\n", " '1064a',\n", - " '1056a',\n", - " '110c',\n", " '889b',\n", - " '106a',\n", - " '132b',\n", " '134e',\n", " '1055a',\n", " '889a',\n", " '95a',\n", - " '133a',\n", " '126a',\n", " '134a',\n", - " '132c',\n", - " '1056c',\n", " '130a',\n", " '623a',\n", " '1373a',\n", - " '116a',\n", - " '132a'],\n", - " 'HLS_L30': ['132a',\n", - " '116a',\n", + " '116a'],\n", + " 'HLS_L30': ['116a',\n", " '106d',\n", - " '95a',\n", " '126b',\n", " '1344a',\n", - " '1373d',\n", - " '110e',\n", " '1378a',\n", " '1338a',\n", - " '110c',\n", - " '132b',\n", " '1376a',\n", " '133a',\n", " '613b',\n", " '915a',\n", - " '132c',\n", " '1052a',\n", - " '116d',\n", " '623d',\n", - " '1373e',\n", " '628a',\n", " '889a',\n", - " '1373a',\n", " '130a',\n", " '126a',\n", " '106a',\n", - " '110a',\n", " '623a',\n", - " '129a',\n", - " '116e']}" + " '129a']}" ] }, - "execution_count": 44, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -1691,7 +1666,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 58, "id": "6f59f9a1", "metadata": {}, "outputs": [ @@ -1710,7 +1685,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 59, "id": "19734cc8", "metadata": {}, "outputs": [], @@ -1736,7 +1711,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 60, "id": "3c4cee14", "metadata": {}, "outputs": [ @@ -1745,14 +1720,66 @@ "output_type": "stream", "text": [ "Chipping: 95a 95a.MASK.tif 95a.HLS_S30.stacked.tif 95a.HLS_S30.2018-07-02.Fmask.tif\n", + "Chipping: 102a 102a.MASK.tif 102a.HLS_S30.stacked.tif 102a.HLS_S30.2019-07-17.Fmask.tif\n", + "Chipping: 106d 106d.MASK.tif 106d.HLS_S30.stacked.tif 106d.HLS_S30.2019-08-28.Fmask.tif\n", "Chipping: 126a 126a.MASK.tif 126a.HLS_S30.stacked.tif 126a.HLS_S30.2018-08-11.Fmask.tif\n", + "Chipping: 126b 126b.MASK.tif 126b.HLS_S30.stacked.tif 126b.HLS_S30.2018-08-11.Fmask.tif\n", + "Chipping: 129a 129a.MASK.tif 129a.HLS_S30.stacked.tif 129a.HLS_S30.2018-08-07.Fmask.tif\n", + "Chipping: 130a 130a.MASK.tif 130a.HLS_S30.stacked.tif 130a.HLS_S30.2018-08-07.Fmask.tif\n", + "Chipping: 134a 134a.MASK.tif 134a.HLS_S30.stacked.tif 134a.HLS_S30.2018-07-12.Fmask.tif\n", + "Chipping: 134e 134e.MASK.tif 134e.HLS_S30.stacked.tif 134e.HLS_S30.2018-07-12.Fmask.tif\n", + "Chipping: 597b 597b.MASK.tif 597b.HLS_S30.stacked.tif 597b.HLS_S30.2017-07-15.Fmask.tif\n", + "Chipping: 623d 623d.MASK.tif 623d.HLS_S30.stacked.tif 623d.HLS_S30.2019-08-19.Fmask.tif\n", + "Chipping: 623a 623a.MASK.tif 623a.HLS_S30.stacked.tif 623a.HLS_S30.2019-08-19.Fmask.tif\n", + "Chipping: 889a 889a.MASK.tif 889a.HLS_S30.stacked.tif 889a.HLS_S30.2018-07-13.Fmask.tif\n", + "Chipping: 889b 889b.MASK.tif 889b.HLS_S30.stacked.tif 889b.HLS_S30.2018-07-13.Fmask.tif\n", + "Chipping: 892a 892a.MASK.tif 892a.HLS_S30.stacked.tif 892a.HLS_S30.2018-07-08.Fmask.tif\n", + "Chipping: 1079a 1079a.MASK.tif 1079a.HLS_S30.stacked.tif 1079a.HLS_S30.2019-08-04.Fmask.tif\n", + "Chipping: 1069a 1069a.MASK.tif 1069a.HLS_S30.stacked.tif 1069a.HLS_S30.2019-08-14.Fmask.tif\n", + "Chipping: 1380a 1380a.MASK.tif 1380a.HLS_S30.stacked.tif 1380a.HLS_S30.2018-06-16.Fmask.tif\n", + "Chipping: 628a 628a.MASK.tif 628a.HLS_S30.stacked.tif 628a.HLS_S30.2019-07-18.Fmask.tif\n", + "Chipping: 638a 638a.MASK.tif 638a.HLS_S30.stacked.tif 638a.HLS_S30.2020-06-12.Fmask.tif\n", + "Chipping: 116d 116d.MASK.tif 116d.HLS_S30.stacked.tif 116d.HLS_S30.2020-06-17.Fmask.tif\n", "Chipping: 116a 116a.MASK.tif 116a.HLS_S30.stacked.tif 116a.HLS_S30.2020-06-17.Fmask.tif\n", "Chipping: 116e 116e.MASK.tif 116e.HLS_S30.stacked.tif 116e.HLS_S30.2020-06-17.Fmask.tif\n", + "Chipping: 648a 648a.MASK.tif 648a.HLS_S30.stacked.tif 648a.HLS_S30.2020-07-16.Fmask.tif\n", + "Chipping: 1055a 1055a.MASK.tif 1055a.HLS_S30.stacked.tif 1055a.HLS_S30.2018-08-04.Fmask.tif\n", + "Chipping: 1347a 1347a.MASK.tif 1347a.HLS_S30.stacked.tif 1347a.HLS_S30.2018-08-04.Fmask.tif\n", + "Chipping: 1064a 1064a.MASK.tif 1064a.HLS_S30.stacked.tif 1064a.HLS_S30.2018-07-03.Fmask.tif\n", + "Chipping: 1338a 1338a.MASK.tif 1338a.HLS_S30.stacked.tif 1338a.HLS_S30.2019-09-05.Fmask.tif\n", + "Chipping: 1373a 1373a.MASK.tif 1373a.HLS_S30.stacked.tif 1373a.HLS_S30.2020-08-17.Fmask.tif\n", + "Chipping: 1373d 1373d.MASK.tif 1373d.HLS_S30.stacked.tif 1373d.HLS_S30.2020-08-17.Fmask.tif\n", + "Chipping: 1373e 1373e.MASK.tif 1373e.HLS_S30.stacked.tif 1373e.HLS_S30.2020-08-17.Fmask.tif\n", "Chipping: 106a 106a.MASK.tif 106a.HLS_L30.stacked.tif 106a.HLS_L30.2019-08-28.fmask.tif\n", + "Chipping: 106d 106d.MASK.tif 106d.HLS_L30.stacked.tif 106d.HLS_L30.2019-08-28.fmask.tif\n", + "Already chipped event 106d: skipping...\n", "Chipping: 126a 126a.MASK.tif 126a.HLS_L30.stacked.tif 126a.HLS_L30.2018-08-11.fmask.tif\n", "Already chipped event 126a: skipping...\n", + "Chipping: 126b 126b.MASK.tif 126b.HLS_L30.stacked.tif 126b.HLS_L30.2018-08-11.fmask.tif\n", + "Already chipped event 126b: skipping...\n", + "Chipping: 129a 129a.MASK.tif 129a.HLS_L30.stacked.tif 129a.HLS_L30.2018-08-07.fmask.tif\n", + "Already chipped event 129a: skipping...\n", + "Chipping: 130a 130a.MASK.tif 130a.HLS_L30.stacked.tif 130a.HLS_L30.2018-08-07.fmask.tif\n", + "Already chipped event 130a: skipping...\n", + "Chipping: 133a 133a.MASK.tif 133a.HLS_L30.stacked.tif 133a.HLS_L30.2018-07-26.fmask.tif\n", + "Chipping: 613b 613b.MASK.tif 613b.HLS_L30.stacked.tif 613b.HLS_L30.2018-07-08.fmask.tif\n", + "Chipping: 623d 623d.MASK.tif 623d.HLS_L30.stacked.tif 623d.HLS_L30.2019-08-19.fmask.tif\n", + "Already chipped event 623d: skipping...\n", + "Chipping: 623a 623a.MASK.tif 623a.HLS_L30.stacked.tif 623a.HLS_L30.2019-08-19.fmask.tif\n", + "Already chipped event 623a: skipping...\n", + "Chipping: 889a 889a.MASK.tif 889a.HLS_L30.stacked.tif 889a.HLS_L30.2018-07-13.fmask.tif\n", + "Already chipped event 889a: skipping...\n", + "Chipping: 915a 915a.MASK.tif 915a.HLS_L30.stacked.tif 915a.HLS_L30.2017-07-26.fmask.tif\n", + "Chipping: 1378a 1378a.MASK.tif 1378a.HLS_L30.stacked.tif 1378a.HLS_L30.2018-07-12.fmask.tif\n", + "Chipping: 628a 628a.MASK.tif 628a.HLS_L30.stacked.tif 628a.HLS_L30.2019-07-18.fmask.tif\n", + "Already chipped event 628a: skipping...\n", "Chipping: 116a 116a.MASK.tif 116a.HLS_L30.stacked.tif 116a.HLS_L30.2020-06-17.fmask.tif\n", - "Already chipped event 116a: skipping...\n" + "Already chipped event 116a: skipping...\n", + "Chipping: 1052a 1052a.MASK.tif 1052a.HLS_L30.stacked.tif 1052a.HLS_L30.2017-08-13.fmask.tif\n", + "Chipping: 1338a 1338a.MASK.tif 1338a.HLS_L30.stacked.tif 1338a.HLS_L30.2019-09-05.fmask.tif\n", + "Already chipped event 1338a: skipping...\n", + "Chipping: 1344a 1344a.MASK.tif 1344a.HLS_L30.stacked.tif 1344a.HLS_L30.2018-08-02.fmask.tif\n", + "Chipping: 1376a 1376a.MASK.tif 1376a.HLS_L30.stacked.tif 1376a.HLS_L30.2020-07-19.fmask.tif\n" ] } ], @@ -1805,17 +1832,17 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 61, "id": "252dff47", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(24, 112)" + "(1, 1544)" ] }, - "execution_count": 47, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -1826,7 +1853,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 62, "id": "6a7dca8f", "metadata": {}, "outputs": [], @@ -1844,17 +1871,17 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 63, "id": "b1c32b82", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "40" + "392" ] }, - "execution_count": 51, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -1890,7 +1917,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 64, "id": "3049d797", "metadata": {}, "outputs": [], @@ -1934,7 +1961,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 65, "id": "00880cff", "metadata": {}, "outputs": [ @@ -1949,7 +1976,7 @@ " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.SW2.tif')]" ] }, - "execution_count": 53, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -1960,7 +1987,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 66, "id": "f68fa713", "metadata": {}, "outputs": [], @@ -2007,14 +2034,6 @@ "\n", "(output_base / 'statistics.json').write_text(json.dumps(stats_file, indent=2))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2aed8fc1", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From b4d6f06a817c39f7a10e6099a2ba58b77d409170 Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 29 Apr 2026 17:53:59 -0800 Subject: [PATCH 21/21] Filter chip by damage/no damage --- notebooks/hwds-example.ipynb | 2042 ++++++++++++++++------------------ src/satchip/chip_data.py | 92 +- 2 files changed, 1021 insertions(+), 1113 deletions(-) diff --git a/notebooks/hwds-example.ipynb b/notebooks/hwds-example.ipynb index 3c08ae4..51d8b10 100644 --- a/notebooks/hwds-example.ipynb +++ b/notebooks/hwds-example.ipynb @@ -2,10 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": 28, + "execution_count": 1, "id": "4dcd6420", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/wbhorn/miniforge3/envs/satchip/lib/python3.14/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from pathlib import Path\n", "import shutil\n", @@ -23,7 +32,7 @@ "metadata": {}, "outputs": [], "source": [ - "shp_path = 'hwds_pristine'\n", + "shp_path = 'Pristine_Merged'\n", "df = gpd.read_file(shp_path)\n", "\n", "df['SwathDate'] = pd.to_datetime(df['SwathDate'], format='%Y-%m-%d')\n", @@ -34,7 +43,507 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 3, + "id": "55da8795", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SwathDateHLSIDMODISIDVisibilityHLSDateFieldgeometry
02019-07-09102a10212019-07-17S2POLYGON Z ((-99.49952 40.26693 0, -99.47639 40...
12019-08-13106a10622019-08-28LSPOLYGON Z ((-101.59283 40.78718 0, -101.60341 ...
22018-08-06126a12612018-08-11S2POLYGON Z ((-98.2416 41.16581 0, -98.24082 41....
32018-08-06127a12712018-08-11S2POLYGON Z ((-97.78549 41.00092 0, -97.76367 41...
42018-07-29129a12932018-08-07LSPOLYGON Z ((-103.1513 43.15378 0, -103.14676 4...
52018-07-27130a13022018-08-07LSPOLYGON Z ((-102.98845 43.56544 0, -102.97929 ...
62018-07-27132a13232018-08-11LSMULTIPOLYGON Z (((-97.42377 41.60069 0, -97.42...
72018-07-18133a13322018-07-26LSPOLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6...
82018-06-30134a13412018-07-12S2POLYGON Z ((-99.72435 40.41203 0, -99.70477 40...
92017-07-05598a59822017-07-15S2POLYGON Z ((-98.60738 45.24939 0, -98.60167 45...
102018-06-26614a61132018-07-08LSMULTIPOLYGON Z (((-100.65633 44.66629 0, -100....
112019-08-06623d62422019-08-19LSPOLYGON Z ((-100.00341 45.55568 0, -99.98419 4...
122019-08-06623a62312019-08-19LSPOLYGON Z ((-100.33919 46.03454 0, -100.33567 ...
132018-07-02889a88922018-07-13S2POLYGON Z ((-102.70434 46.61383 0, -102.68865 ...
142018-07-02889b89022018-07-13S2POLYGON Z ((-101.87838 46.45158 0, -101.8724 4...
152018-06-26892a89222018-07-08S2POLYGON Z ((-103.21657 46.25444 0, -103.21434 ...
162017-07-20915a91532017-07-26LSPOLYGON Z ((-102.58639 47.46329 0, -102.55608 ...
172019-07-291079a107922019-08-04S2POLYGON Z ((-102.59566 39.43458 0, -102.60361 ...
182019-08-131069a106932019-08-14S2POLYGON Z ((-102.15607 39.11807 0, -102.14741 ...
192018-07-101378a137832018-07-12LSPOLYGON Z ((-95.05656 43.17247 0, -95.04818 43...
202019-07-03628a62832019-07-13S2POLYGON Z ((-102.31279 43.36328 0, -102.30699 ...
212020-06-04638a63832020-06-12S2POLYGON Z ((-103.23338 45.55551 0, -103.21165 ...
222020-06-07116d63932020-06-17S2POLYGON Z ((-101.18151 43.22672 0, -101.17656 ...
232020-06-07116e64022020-06-17S2POLYGON Z ((-100.67301 43.4943 0, -100.68216 4...
242020-07-06648a64822020-07-16S2POLYGON Z ((-97.35114 43.00508 0, -97.34554 43...
252017-08-101052a105222017-08-13LSPOLYGON Z ((-102.68889 38.15458 0, -102.68788 ...
262018-07-261055a105522018-08-04S2POLYGON Z ((-102.09648 39.57415 0, -102.09107 ...
272018-07-281056a105632018-08-07S2MULTIPOLYGON Z (((-101.94001 40.61807 0, -101....
282018-07-281347a134722018-08-04S2POLYGON Z ((-100.73413 39.35341 0, -100.73135 ...
292018-06-191064a106422018-07-03S2POLYGON Z ((-102.42969 39.8437 0, -102.42208 3...
302019-08-241338a133832019-09-05S2POLYGON Z ((-100.22342 39.23137 0, -100.20831 ...
312018-07-271344a134422018-08-02S2POLYGON Z ((-99.62458 38.06997 0, -99.61494 38...
322020-08-101373a137312020-08-17S2POLYGON Z ((-94.97657 42.20582 0, -94.97333 42...
332020-08-101373d137332020-08-17S2POLYGON Z ((-94.80415 42.01749 0, -94.78768 42...
342020-08-101373e137332020-08-17S2POLYGON Z ((-94.55871 41.9804 0, -94.53908 41....
352020-07-111376a137612020-07-19LSPOLYGON Z ((-92.15179 43.02718 0, -92.1428 43....
362019-08-22110d11032019-09-05S2MULTIPOLYGON Z (((-98.39133 40.49204 0, -98.39...
\n", + "
" + ], + "text/plain": [ + " SwathDate HLSID MODISID Visibility HLSDate Field \\\n", + "0 2019-07-09 102a 102 1 2019-07-17 S2 \n", + "1 2019-08-13 106a 106 2 2019-08-28 LS \n", + "2 2018-08-06 126a 126 1 2018-08-11 S2 \n", + "3 2018-08-06 127a 127 1 2018-08-11 S2 \n", + "4 2018-07-29 129a 129 3 2018-08-07 LS \n", + "5 2018-07-27 130a 130 2 2018-08-07 LS \n", + "6 2018-07-27 132a 132 3 2018-08-11 LS \n", + "7 2018-07-18 133a 133 2 2018-07-26 LS \n", + "8 2018-06-30 134a 134 1 2018-07-12 S2 \n", + "9 2017-07-05 598a 598 2 2017-07-15 S2 \n", + "10 2018-06-26 614a 611 3 2018-07-08 LS \n", + "11 2019-08-06 623d 624 2 2019-08-19 LS \n", + "12 2019-08-06 623a 623 1 2019-08-19 LS \n", + "13 2018-07-02 889a 889 2 2018-07-13 S2 \n", + "14 2018-07-02 889b 890 2 2018-07-13 S2 \n", + "15 2018-06-26 892a 892 2 2018-07-08 S2 \n", + "16 2017-07-20 915a 915 3 2017-07-26 LS \n", + "17 2019-07-29 1079a 1079 2 2019-08-04 S2 \n", + "18 2019-08-13 1069a 1069 3 2019-08-14 S2 \n", + "19 2018-07-10 1378a 1378 3 2018-07-12 LS \n", + "20 2019-07-03 628a 628 3 2019-07-13 S2 \n", + "21 2020-06-04 638a 638 3 2020-06-12 S2 \n", + "22 2020-06-07 116d 639 3 2020-06-17 S2 \n", + "23 2020-06-07 116e 640 2 2020-06-17 S2 \n", + "24 2020-07-06 648a 648 2 2020-07-16 S2 \n", + "25 2017-08-10 1052a 1052 2 2017-08-13 LS \n", + "26 2018-07-26 1055a 1055 2 2018-08-04 S2 \n", + "27 2018-07-28 1056a 1056 3 2018-08-07 S2 \n", + "28 2018-07-28 1347a 1347 2 2018-08-04 S2 \n", + "29 2018-06-19 1064a 1064 2 2018-07-03 S2 \n", + "30 2019-08-24 1338a 1338 3 2019-09-05 S2 \n", + "31 2018-07-27 1344a 1344 2 2018-08-02 S2 \n", + "32 2020-08-10 1373a 1373 1 2020-08-17 S2 \n", + "33 2020-08-10 1373d 1373 3 2020-08-17 S2 \n", + "34 2020-08-10 1373e 1373 3 2020-08-17 S2 \n", + "35 2020-07-11 1376a 1376 1 2020-07-19 LS \n", + "36 2019-08-22 110d 110 3 2019-09-05 S2 \n", + "\n", + " geometry \n", + "0 POLYGON Z ((-99.49952 40.26693 0, -99.47639 40... \n", + "1 POLYGON Z ((-101.59283 40.78718 0, -101.60341 ... \n", + "2 POLYGON Z ((-98.2416 41.16581 0, -98.24082 41.... \n", + "3 POLYGON Z ((-97.78549 41.00092 0, -97.76367 41... \n", + "4 POLYGON Z ((-103.1513 43.15378 0, -103.14676 4... \n", + "5 POLYGON Z ((-102.98845 43.56544 0, -102.97929 ... \n", + "6 MULTIPOLYGON Z (((-97.42377 41.60069 0, -97.42... \n", + "7 POLYGON Z ((-97.5217 42.65295 0, -97.5154 42.6... \n", + "8 POLYGON Z ((-99.72435 40.41203 0, -99.70477 40... \n", + "9 POLYGON Z ((-98.60738 45.24939 0, -98.60167 45... \n", + "10 MULTIPOLYGON Z (((-100.65633 44.66629 0, -100.... \n", + "11 POLYGON Z ((-100.00341 45.55568 0, -99.98419 4... \n", + "12 POLYGON Z ((-100.33919 46.03454 0, -100.33567 ... \n", + "13 POLYGON Z ((-102.70434 46.61383 0, -102.68865 ... \n", + "14 POLYGON Z ((-101.87838 46.45158 0, -101.8724 4... \n", + "15 POLYGON Z ((-103.21657 46.25444 0, -103.21434 ... \n", + "16 POLYGON Z ((-102.58639 47.46329 0, -102.55608 ... \n", + "17 POLYGON Z ((-102.59566 39.43458 0, -102.60361 ... \n", + "18 POLYGON Z ((-102.15607 39.11807 0, -102.14741 ... \n", + "19 POLYGON Z ((-95.05656 43.17247 0, -95.04818 43... \n", + "20 POLYGON Z ((-102.31279 43.36328 0, -102.30699 ... \n", + "21 POLYGON Z ((-103.23338 45.55551 0, -103.21165 ... \n", + "22 POLYGON Z ((-101.18151 43.22672 0, -101.17656 ... \n", + "23 POLYGON Z ((-100.67301 43.4943 0, -100.68216 4... \n", + "24 POLYGON Z ((-97.35114 43.00508 0, -97.34554 43... \n", + "25 POLYGON Z ((-102.68889 38.15458 0, -102.68788 ... \n", + "26 POLYGON Z ((-102.09648 39.57415 0, -102.09107 ... \n", + "27 MULTIPOLYGON Z (((-101.94001 40.61807 0, -101.... \n", + "28 POLYGON Z ((-100.73413 39.35341 0, -100.73135 ... \n", + "29 POLYGON Z ((-102.42969 39.8437 0, -102.42208 3... \n", + "30 POLYGON Z ((-100.22342 39.23137 0, -100.20831 ... \n", + "31 POLYGON Z ((-99.62458 38.06997 0, -99.61494 38... \n", + "32 POLYGON Z ((-94.97657 42.20582 0, -94.97333 42... \n", + "33 POLYGON Z ((-94.80415 42.01749 0, -94.78768 42... \n", + "34 POLYGON Z ((-94.55871 41.9804 0, -94.53908 41.... \n", + "35 POLYGON Z ((-92.15179 43.02718 0, -92.1428 43.... \n", + "36 MULTIPOLYGON Z (((-98.39133 40.49204 0, -98.39... " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "id": "f1a32464", "metadata": {}, "outputs": [ @@ -75,7 +584,7 @@ " 'plots': PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/plots')}}" ] }, - "execution_count": 25, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -88,1397 +597,674 @@ "\n", " return {\n", " 'modality': modality,\n", - " 'raw': base_path / 'raw',\n", - " 'merged': base_path / 'merged',\n", - " 'wgs84': base_path / 'wgs84',\n", - " 'stacked': base_path / 'stacked',\n", - " 'warped': base_path / 'warped',\n", - " 'chips': base_path / 'chips',\n", - " 'plots': base_path / 'plots'\n", - " }\n", - "\n", - "mod_paths = {\n", - " modality['id']: make_mod_paths(modality) for modality in (models.HLS_S30, models.HLS_L30)\n", - "}\n", - "mod_paths" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d0202e9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/raw\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11050.57it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 280659.75it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 397355.12it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 95a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/95a.MASK.tif\n", - "Adding event files for 95a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 11599.83it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 461758.24it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 766471.80it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 102a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/102a.MASK.tif\n", - "Adding event files for 102a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 31564.69it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 637109.47it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 108/108 [00:00<00:00, 884736.00it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 106a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/106a.MASK.tif\n", - "Adding event files for 106a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12146.65it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 450731.18it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 786432.00it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 110a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110a.MASK.tif\n", - "Adding event files for 110a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11597.15it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 335544.32it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 496693.89it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 110c\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110c.MASK.tif\n", - "Adding event files for 110c\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10169.38it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 337042.29it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 490243.32it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 106d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/106d.MASK.tif\n", - "Adding event files for 106d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 9952.21it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 242757.14it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 408094.44it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 110g\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110g.MASK.tif\n", - "Adding event files for 110g\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13540.93it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 355282.22it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 820624.70it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 110e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110e.MASK.tif\n", - "Adding event files for 110e\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10591.68it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 267721.53it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 298408.98it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 126a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/126a.MASK.tif\n", - "Adding event files for 126a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13703.14it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 499983.26it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 477832.10it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 126b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/126b.MASK.tif\n", - "Adding event files for 126b\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 11725.94it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 410312.35it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 665175.96it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 129a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/129a.MASK.tif\n", - "Adding event files for 129a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 23899.17it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 715615.85it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 917902.40it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 130a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/130a.MASK.tif\n", - "Adding event files for 130a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 22414.45it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 713924.09it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 829642.55it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 132a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132a.MASK.tif\n", - "Adding event files for 132a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12601.81it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 473338.38it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 356962.04it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 132b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132b.MASK.tif\n", - "Adding event files for 132b\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14285.24it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 503316.48it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 766471.80it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 132c\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/132c.MASK.tif\n", - "Adding event files for 132c\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13193.09it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 438938.79it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 751218.63it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 133a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/133a.MASK.tif\n", - "Adding event files for 133a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13699.41it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 488656.78it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 743817.46it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 134a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/134a.MASK.tif\n", - "Adding event files for 134a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 26786.40it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 720739.59it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 971028.58it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 134e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/134e.MASK.tif\n", - "Adding event files for 134e\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 15107.05it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 518882.97it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 909608.10it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 597b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/597b.MASK.tif\n", - "Adding event files for 597b\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24889.96it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 634432.54it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1184274.07it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 611a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/611a.MASK.tif\n", - "Adding event files for 611a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13246.33it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 420598.73it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 613b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/613b.MASK.tif\n", - "Adding event files for 613b\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13540.93it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 376546.00it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 816188.89it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 623d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/623d.MASK.tif\n", - "Adding event files for 623d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 20809.67it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 580749.78it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1006632.96it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 623a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/623a.MASK.tif\n", - "Adding event files for 623a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12841.89it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 496693.89it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 754974.72it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 889a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889a.MASK.tif\n", - "Adding event files for 889a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12384.76it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 522473.85it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 816188.89it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 889b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889b.MASK.tif\n", - "Adding event files for 889b\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13572.58it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 520672.22it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 892a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/892a.MASK.tif\n", - "Adding event files for 892a\n", - "no HLS_S30 data for 915a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 49201.83it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 749148.01it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 162/162 [00:00<00:00, 1189977.67it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1079a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1079a.MASK.tif\n", - "Adding event files for 1079a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12855.01it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 308152.95it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 848286.20it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1069a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1069a.MASK.tif\n", - "Adding event files for 1069a\n", - "no HLS_S30 data for 1378a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11163.31it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 235194.62it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 510118.05it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1380a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1380a.MASK.tif\n", - "Adding event files for 1380a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24995.02it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 601573.48it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1016800.97it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 628a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/628a.MASK.tif\n", - "Adding event files for 628a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 27135.89it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 646382.47it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1037053.19it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 638a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/638a.MASK.tif\n", - "Adding event files for 638a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 27245.57it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 681692.75it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1090216.20it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 116d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116d.MASK.tif\n", - "Adding event files for 116d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14768.68it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 499983.26it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 441505.68it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 116a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116a.MASK.tif\n", - "Adding event files for 116a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12733.59it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 452080.67it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 736560.70it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 116e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116e.MASK.tif\n", - "Adding event files for 116e\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10817.81it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 344737.32it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 487080.46it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 648a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/648a.MASK.tif\n", - "Adding event files for 648a\n", - "no HLS_S30 data for 1052a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 34015.53it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 793318.44it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 108/108 [00:00<00:00, 1402429.82it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1055a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1055a.MASK.tif\n", - "Adding event files for 1055a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 24254.27it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 633102.49it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1070886.13it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1056a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1056a.MASK.tif\n", - "Adding event files for 1056a\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 27889.72it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 456178.08it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1110256.94it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found HLS_S30 data for 1056c\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1056c.MASK.tif\n", - "Adding event files for 1056c\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13274.28it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 412554.49it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 867787.03it/s]\n" - ] - }, + " 'raw': base_path / 'raw',\n", + " 'merged': base_path / 'merged',\n", + " 'wgs84': base_path / 'wgs84',\n", + " 'stacked': base_path / 'stacked',\n", + " 'warped': base_path / 'warped',\n", + " 'chips': base_path / 'chips',\n", + " 'plots': base_path / 'plots'\n", + " }\n", + "\n", + "mod_paths = {\n", + " modality['id']: make_mod_paths(modality) for modality in (models.HLS_S30, models.HLS_L30)\n", + "}\n", + "mod_paths" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0d0202e9", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1347a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1347a.MASK.tif\n", - "Adding event files for 1347a\n" + "{'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/raw\n", + "Logging in to earthaccess\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 23226.42it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 681692.75it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1139584.48it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13296.49it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 371908.73it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 725937.23it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1064a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1064a.MASK.tif\n", - "Adding event files for 1064a\n" + "Found HLS_S30 data for 102a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/102a.MASK.tif\n", + "Adding event files for 102a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13982.31it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 503316.48it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 898779.43it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11018.31it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 324023.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 431414.13it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1338a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1338a.MASK.tif\n", - "Adding event files for 1338a\n", - "no HLS_S30 data for 1344a\n" + "Found HLS_S30 data for 126a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/126a.MASK.tif\n", + "Adding event files for 126a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11209.72it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 204047.22it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 414821.27it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12883.53it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 474826.87it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 848286.20it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1373a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373a.MASK.tif\n", - "Adding event files for 1373a\n" + "Found HLS_S30 data for 127a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/127a.MASK.tif\n", + "Adding event files for 127a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10433.59it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 324023.48it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 513588.24it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14230.04it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 547083.13it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 811800.77it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1373d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373d.MASK.tif\n", - "Adding event files for 1373d\n" + "Found HLS_S30 data for 134a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/134a.MASK.tif\n", + "Adding event files for 134a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13342.31it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 482411.96it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 662258.53it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13996.57it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 453438.27it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 715615.85it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_S30 data for 1373e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373e.MASK.tif\n", - "Adding event files for 1373e\n", - "no HLS_S30 data for 1376a\n", - "{'id': 'HLS_L30', 'collection': 'HLSL30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B05', name='NIR Narrow', shortname='N'), Band(id='B06', name='SWIR 1', shortname='SW1'), Band(id='B07', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/raw\n" + "Found HLS_S30 data for 598a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/598a.MASK.tif\n", + "Adding event files for 598a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10092.17it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 261056.27it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 331129.26it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14614.30it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 401582.30it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 695829.24it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 95a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/95a.MASK.tif\n", - "Adding event files for 95a\n", - "no HLS_L30 data for 102a\n" + "Found HLS_S30 data for 889a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889a.MASK.tif\n", + "Adding event files for 889a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 19294.01it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 737280.00it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1301680.55it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13644.94it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 461758.24it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 106a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/106a.MASK.tif\n", - "Adding event files for 106a\n" + "Found HLS_S30 data for 889b\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/889b.MASK.tif\n", + "Adding event files for 889b\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12599.29it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 441505.68it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 753467.78it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 26597.66it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 677107.37it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1263556.02it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 110a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110a.MASK.tif\n", - "Adding event files for 110a\n" + "Found HLS_S30 data for 892a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/892a.MASK.tif\n", + "Adding event files for 892a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 9554.22it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 288598.90it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 439961.96it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 50027.78it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 162/162 [00:00<00:00, 109610.78it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 162/162 [00:00<00:00, 1085426.91it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 110c\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110c.MASK.tif\n", - "Adding event files for 110c\n" + "Found HLS_S30 data for 1079a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1079a.MASK.tif\n", + "Adding event files for 1079a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10265.06it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 186690.09it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 433893.52it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13109.48it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 531672.34it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 904161.34it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 106d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/106d.MASK.tif\n", - "Adding event files for 106d\n", - "no HLS_L30 data for 110g\n" + "Found HLS_S30 data for 1069a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1069a.MASK.tif\n", + "Adding event files for 1069a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13089.47it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 443060.28it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 28225.99it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 702302.07it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1090216.20it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 110e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/110e.MASK.tif\n", - "Adding event files for 110e\n" + "Found HLS_S30 data for 628a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/628a.MASK.tif\n", + "Adding event files for 628a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10765.67it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 268865.64it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 301026.60it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 28299.52it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 768813.36it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1362770.25it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 126a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/126a.MASK.tif\n", - "Adding event files for 126a\n" + "Found HLS_S30 data for 638a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/638a.MASK.tif\n", + "Adding event files for 638a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13122.24it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 441505.68it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 776722.96it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 25067.64it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 610080.58it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1212810.80it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 126b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/126b.MASK.tif\n", - "Adding event files for 126b\n" + "Found HLS_S30 data for 116d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116d.MASK.tif\n", + "Adding event files for 116d\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12715.15it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 446202.55it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 744551.01it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 12609.18it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 533551.04it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 461758.24it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 129a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/129a.MASK.tif\n", - "Adding event files for 129a\n" + "Found HLS_S30 data for 116e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/116e.MASK.tif\n", + "Adding event files for 116e\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 16514.09it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 699050.67it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 975419.53it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 11193.10it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 357807.92it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 426539.39it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 130a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/130a.MASK.tif\n", - "Adding event files for 130a\n" + "Found HLS_S30 data for 648a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/648a.MASK.tif\n", + "Adding event files for 648a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 21204.77it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 621378.37it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1027176.49it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 39645.09it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 108/108 [00:00<00:00, 120731.57it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 108/108 [00:00<00:00, 1438047.09it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 132a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132a.MASK.tif\n", - "Adding event files for 132a\n" + "Found HLS_S30 data for 1055a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1055a.MASK.tif\n", + "Adding event files for 1055a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13877.70it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 432402.47it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 467766.25it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 27267.71it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 72/72 [00:00<00:00, 774333.05it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 72/72 [00:00<00:00, 1247892.10it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 132b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132b.MASK.tif\n", - "Adding event files for 132b\n" + "Found HLS_S30 data for 1056a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1056a.MASK.tif\n", + "Adding event files for 1056a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 10378.52it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 408536.10it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 577197.80it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13693.20it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 517105.97it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 848286.20it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 132c\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132c.MASK.tif\n", - "Adding event files for 132c\n" + "Found HLS_S30 data for 1347a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1347a.MASK.tif\n", + "Adding event files for 1347a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 14810.40it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 454256.75it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 31324.15it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 498662.30it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1310720.00it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 133a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/133a.MASK.tif\n", - "Adding event files for 133a\n", - "no HLS_L30 data for 134a\n", - "no HLS_L30 data for 134e\n", - "no HLS_L30 data for 597b\n", - "no HLS_L30 data for 611a\n" + "Found HLS_S30 data for 1064a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1064a.MASK.tif\n", + "Adding event files for 1064a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12875.18it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 435394.88it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 801459.36it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14260.95it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 601573.48it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 1027176.49it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 613b\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/613b.MASK.tif\n", - "Adding event files for 613b\n" + "Found HLS_S30 data for 1338a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1338a.MASK.tif\n", + "Adding event files for 1338a\n", + "no HLS_S30 data for 1344a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11924.67it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 427990.20it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 629145.60it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 646.08it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 235194.62it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 351151.03it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 623d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623d.MASK.tif\n", - "Adding event files for 623d\n" + "Found HLS_S30 data for 1373a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373a.MASK.tif\n", + "Adding event files for 1373a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 20947.08it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 640351.76it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 967916.31it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 10624.47it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 18/18 [00:00<00:00, 387166.52it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 18/18 [00:00<00:00, 585251.72it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 623a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623a.MASK.tif\n", - "Adding event files for 623a\n" + "Found HLS_S30 data for 1373d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373d.MASK.tif\n", + "Adding event files for 1373d\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11818.27it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 449389.71it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 758006.75it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 14273.08it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 431414.13it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 834226.21it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 889a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/889a.MASK.tif\n", - "Adding event files for 889a\n", - "no HLS_L30 data for 889b\n", - "no HLS_L30 data for 892a\n" + "Found HLS_S30 data for 1373e\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/1373e.MASK.tif\n", + "Adding event files for 1373e\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13824.34it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 435394.88it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 571950.55it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 13669.65it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 36/36 [00:00<00:00, 506694.44it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 36/36 [00:00<00:00, 555128.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 915a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/915a.MASK.tif\n", - "Adding event files for 915a\n", - "no HLS_L30 data for 1079a\n", - "no HLS_L30 data for 1069a\n" + "Found HLS_S30 data for 110d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/wgs84/110d.MASK.tif\n", + "Adding event files for 110d\n", + "{'id': 'HLS_L30', 'collection': 'HLSL30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B05', name='NIR Narrow', shortname='N'), Band(id='B06', name='SWIR 1', shortname='SW1'), Band(id='B07', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='fmask'))} /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/raw\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 10954.05it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 462607.06it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 702955.98it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 29262.59it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 90/90 [00:00<00:00, 778324.45it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 90/90 [00:00<00:00, 1297207.42it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1378a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1378a.MASK.tif\n", - "Adding event files for 1378a\n", - "no HLS_L30 data for 1380a\n" + "Found HLS_L30 data for 106a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/106a.MASK.tif\n", + "Adding event files for 106a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 21445.10it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 626015.52it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 939023.28it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12653.77it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 469511.64it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 781547.33it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 628a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/628a.MASK.tif\n", - "Adding event files for 628a\n", - "no HLS_L30 data for 638a\n" + "Found HLS_L30 data for 129a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/129a.MASK.tif\n", + "Adding event files for 129a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 20717.73it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 624462.13it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1031386.23it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 22708.74it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 710898.98it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1084733.79it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 116d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116d.MASK.tif\n", - "Adding event files for 116d\n" + "Found HLS_L30 data for 130a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/130a.MASK.tif\n", + "Adding event files for 130a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12356.78it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 354448.23it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 658791.20it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 19331.56it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 700997.88it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1048576.00it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 116a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116a.MASK.tif\n", - "Adding event files for 116a\n" + "Found HLS_L30 data for 132a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/132a.MASK.tif\n", + "Adding event files for 132a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12492.96it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 422245.37it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 635500.61it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 24160.74it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 9597.22it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 758006.75it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 116e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/116e.MASK.tif\n", - "Adding event files for 116e\n", - "no HLS_L30 data for 648a\n" + "Found HLS_L30 data for 133a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/133a.MASK.tif\n", + "Adding event files for 133a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12104.77it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 375609.31it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 714938.18it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 33367.57it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 37803.55it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 1103764.21it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1052a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1052a.MASK.tif\n", - "Adding event files for 1052a\n", - "no HLS_L30 data for 1055a\n", - "no HLS_L30 data for 1056a\n", - "no HLS_L30 data for 1056c\n", - "no HLS_L30 data for 1347a\n", - "no HLS_L30 data for 1064a\n" + "Found HLS_L30 data for 614a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/614a.MASK.tif\n", + "Adding event files for 614a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 11621.79it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 446202.55it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 727335.95it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12575.37it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 443060.28it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 610820.97it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1338a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1338a.MASK.tif\n", - "Adding event files for 1338a\n" + "Found HLS_L30 data for 623d\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623d.MASK.tif\n", + "Adding event files for 623d\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12651.23it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 460912.53it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 436906.67it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 21969.29it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 60/60 [00:00<00:00, 706905.17it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 60/60 [00:00<00:00, 793874.57it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1344a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1344a.MASK.tif\n", - "Adding event files for 1344a\n" + "Found HLS_L30 data for 623a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/623a.MASK.tif\n", + "Adding event files for 623a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 9725.55it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 272357.40it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 343795.41it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12608.13it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 430921.64it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 511500.49it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1373a\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373a.MASK.tif\n", - "Adding event files for 1373a\n" + "Found HLS_L30 data for 915a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/915a.MASK.tif\n", + "Adding event files for 915a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 11376.95it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 283398.92it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 413911.58it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13312.43it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 400729.68it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 744551.01it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1373d\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373d.MASK.tif\n", - "Adding event files for 1373d\n" + "Found HLS_L30 data for 1378a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1378a.MASK.tif\n", + "Adding event files for 1378a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 12795.31it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 455902.61it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 619847.88it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 13177.20it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 30/30 [00:00<00:00, 418036.94it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 30/30 [00:00<00:00, 748982.86it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found HLS_L30 data for 1373e\n", - "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1373e.MASK.tif\n", - "Adding event files for 1373e\n" + "Found HLS_L30 data for 1052a\n", + "generated: /home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/wgs84/1052a.MASK.tif\n", + "Adding event files for 1052a\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 10602.39it/s]\n", - "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 266587.12it/s]\n", - "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 473041.80it/s]\n" + "QUEUEING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 11670.29it/s]\n", + "PROCESSING TASKS | : 100%|██████████| 15/15 [00:00<00:00, 322638.77it/s]\n", + "COLLECTING RESULTS | : 100%|██████████| 15/15 [00:00<00:00, 491520.00it/s]\n" ] }, { @@ -1499,6 +1285,9 @@ "\n", " for idx, row in df.iterrows():\n", " modality = paths['modality']\n", + " best_mod = models.HLS_L30 if row['Field'] == 'LS' else models.HLS_S30\n", + " if modality != best_mod:\n", + " continue\n", "\n", " damage_event = models.Event(\n", " name=row['HLSID'], \n", @@ -1557,7 +1346,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "ed4d4521", "metadata": {}, "outputs": [], @@ -1573,7 +1362,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 8, "id": "519b634b", "metadata": {}, "outputs": [], @@ -1584,12 +1373,12 @@ " plots_backup = plots.parent / 'plots-backup'\n", " shutil.copytree(plots_backup, plots, dirs_exist_ok=True)\n", "\n", - "restore_plots()" + "# restore_plots()" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 9, "id": "44902869", "metadata": {}, "outputs": [ @@ -1598,57 +1387,43 @@ "text/plain": [ "{'HLS_S30': ['628a',\n", " '1079a',\n", - " '597b',\n", " '1069a',\n", - " '1380a',\n", " '116d',\n", " '648a',\n", - " '129a',\n", - " '126b',\n", " '638a',\n", " '116e',\n", " '1338a',\n", + " '598a',\n", " '1347a',\n", " '102a',\n", - " '106d',\n", + " '110d',\n", " '1373e',\n", - " '623d',\n", " '892a',\n", " '1373d',\n", " '1064a',\n", + " '1056a',\n", " '889b',\n", - " '134e',\n", " '1055a',\n", " '889a',\n", - " '95a',\n", " '126a',\n", " '134a',\n", - " '130a',\n", - " '623a',\n", - " '1373a',\n", - " '116a'],\n", - " 'HLS_L30': ['116a',\n", - " '106d',\n", - " '126b',\n", - " '1344a',\n", + " '127a',\n", + " '1373a'],\n", + " 'HLS_L30': ['132a',\n", " '1378a',\n", - " '1338a',\n", " '1376a',\n", " '133a',\n", - " '613b',\n", " '915a',\n", " '1052a',\n", " '623d',\n", - " '628a',\n", - " '889a',\n", " '130a',\n", - " '126a',\n", + " '614a',\n", " '106a',\n", " '623a',\n", " '129a']}" ] }, - "execution_count": 57, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1666,7 +1441,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 10, "id": "6f59f9a1", "metadata": {}, "outputs": [ @@ -1674,7 +1449,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "(Event(name='95a', date=Timestamp('2018-07-02 00:00:00'), wgs84_geometry=, buffer_m=10000), {'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}, (PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.MASK.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.HLS_S30.stacked.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/95a.HLS_S30.2018-07-02.Fmask.tif')))\n" + "(Event(name='102a', date=Timestamp('2019-07-17 00:00:00'), wgs84_geometry=, buffer_m=10000), {'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}, (PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/102a.MASK.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/102a.HLS_S30.stacked.tif'), PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_S30/warped/102a.HLS_S30.2019-07-17.Fmask.tif')))\n" ] } ], @@ -1685,7 +1460,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 11, "id": "19734cc8", "metadata": {}, "outputs": [], @@ -1711,7 +1486,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 12, "id": "3c4cee14", "metadata": {}, "outputs": [ @@ -1719,74 +1494,48 @@ "name": "stdout", "output_type": "stream", "text": [ - "Chipping: 95a 95a.MASK.tif 95a.HLS_S30.stacked.tif 95a.HLS_S30.2018-07-02.Fmask.tif\n", "Chipping: 102a 102a.MASK.tif 102a.HLS_S30.stacked.tif 102a.HLS_S30.2019-07-17.Fmask.tif\n", - "Chipping: 106d 106d.MASK.tif 106d.HLS_S30.stacked.tif 106d.HLS_S30.2019-08-28.Fmask.tif\n", "Chipping: 126a 126a.MASK.tif 126a.HLS_S30.stacked.tif 126a.HLS_S30.2018-08-11.Fmask.tif\n", - "Chipping: 126b 126b.MASK.tif 126b.HLS_S30.stacked.tif 126b.HLS_S30.2018-08-11.Fmask.tif\n", - "Chipping: 129a 129a.MASK.tif 129a.HLS_S30.stacked.tif 129a.HLS_S30.2018-08-07.Fmask.tif\n", - "Chipping: 130a 130a.MASK.tif 130a.HLS_S30.stacked.tif 130a.HLS_S30.2018-08-07.Fmask.tif\n", + "Chipping: 127a 127a.MASK.tif 127a.HLS_S30.stacked.tif 127a.HLS_S30.2018-08-11.Fmask.tif\n", "Chipping: 134a 134a.MASK.tif 134a.HLS_S30.stacked.tif 134a.HLS_S30.2018-07-12.Fmask.tif\n", - "Chipping: 134e 134e.MASK.tif 134e.HLS_S30.stacked.tif 134e.HLS_S30.2018-07-12.Fmask.tif\n", - "Chipping: 597b 597b.MASK.tif 597b.HLS_S30.stacked.tif 597b.HLS_S30.2017-07-15.Fmask.tif\n", - "Chipping: 623d 623d.MASK.tif 623d.HLS_S30.stacked.tif 623d.HLS_S30.2019-08-19.Fmask.tif\n", - "Chipping: 623a 623a.MASK.tif 623a.HLS_S30.stacked.tif 623a.HLS_S30.2019-08-19.Fmask.tif\n", + "Chipping: 598a 598a.MASK.tif 598a.HLS_S30.stacked.tif 598a.HLS_S30.2017-07-15.Fmask.tif\n", "Chipping: 889a 889a.MASK.tif 889a.HLS_S30.stacked.tif 889a.HLS_S30.2018-07-13.Fmask.tif\n", "Chipping: 889b 889b.MASK.tif 889b.HLS_S30.stacked.tif 889b.HLS_S30.2018-07-13.Fmask.tif\n", "Chipping: 892a 892a.MASK.tif 892a.HLS_S30.stacked.tif 892a.HLS_S30.2018-07-08.Fmask.tif\n", "Chipping: 1079a 1079a.MASK.tif 1079a.HLS_S30.stacked.tif 1079a.HLS_S30.2019-08-04.Fmask.tif\n", "Chipping: 1069a 1069a.MASK.tif 1069a.HLS_S30.stacked.tif 1069a.HLS_S30.2019-08-14.Fmask.tif\n", - "Chipping: 1380a 1380a.MASK.tif 1380a.HLS_S30.stacked.tif 1380a.HLS_S30.2018-06-16.Fmask.tif\n", - "Chipping: 628a 628a.MASK.tif 628a.HLS_S30.stacked.tif 628a.HLS_S30.2019-07-18.Fmask.tif\n", + "Chipping: 628a 628a.MASK.tif 628a.HLS_S30.stacked.tif 628a.HLS_S30.2019-07-13.Fmask.tif\n", "Chipping: 638a 638a.MASK.tif 638a.HLS_S30.stacked.tif 638a.HLS_S30.2020-06-12.Fmask.tif\n", "Chipping: 116d 116d.MASK.tif 116d.HLS_S30.stacked.tif 116d.HLS_S30.2020-06-17.Fmask.tif\n", - "Chipping: 116a 116a.MASK.tif 116a.HLS_S30.stacked.tif 116a.HLS_S30.2020-06-17.Fmask.tif\n", "Chipping: 116e 116e.MASK.tif 116e.HLS_S30.stacked.tif 116e.HLS_S30.2020-06-17.Fmask.tif\n", "Chipping: 648a 648a.MASK.tif 648a.HLS_S30.stacked.tif 648a.HLS_S30.2020-07-16.Fmask.tif\n", "Chipping: 1055a 1055a.MASK.tif 1055a.HLS_S30.stacked.tif 1055a.HLS_S30.2018-08-04.Fmask.tif\n", + "Chipping: 1056a 1056a.MASK.tif 1056a.HLS_S30.stacked.tif 1056a.HLS_S30.2018-08-07.Fmask.tif\n", "Chipping: 1347a 1347a.MASK.tif 1347a.HLS_S30.stacked.tif 1347a.HLS_S30.2018-08-04.Fmask.tif\n", "Chipping: 1064a 1064a.MASK.tif 1064a.HLS_S30.stacked.tif 1064a.HLS_S30.2018-07-03.Fmask.tif\n", "Chipping: 1338a 1338a.MASK.tif 1338a.HLS_S30.stacked.tif 1338a.HLS_S30.2019-09-05.Fmask.tif\n", "Chipping: 1373a 1373a.MASK.tif 1373a.HLS_S30.stacked.tif 1373a.HLS_S30.2020-08-17.Fmask.tif\n", "Chipping: 1373d 1373d.MASK.tif 1373d.HLS_S30.stacked.tif 1373d.HLS_S30.2020-08-17.Fmask.tif\n", "Chipping: 1373e 1373e.MASK.tif 1373e.HLS_S30.stacked.tif 1373e.HLS_S30.2020-08-17.Fmask.tif\n", + "Chipping: 110d 110d.MASK.tif 110d.HLS_S30.stacked.tif 110d.HLS_S30.2019-09-05.Fmask.tif\n", "Chipping: 106a 106a.MASK.tif 106a.HLS_L30.stacked.tif 106a.HLS_L30.2019-08-28.fmask.tif\n", - "Chipping: 106d 106d.MASK.tif 106d.HLS_L30.stacked.tif 106d.HLS_L30.2019-08-28.fmask.tif\n", - "Already chipped event 106d: skipping...\n", - "Chipping: 126a 126a.MASK.tif 126a.HLS_L30.stacked.tif 126a.HLS_L30.2018-08-11.fmask.tif\n", - "Already chipped event 126a: skipping...\n", - "Chipping: 126b 126b.MASK.tif 126b.HLS_L30.stacked.tif 126b.HLS_L30.2018-08-11.fmask.tif\n", - "Already chipped event 126b: skipping...\n", "Chipping: 129a 129a.MASK.tif 129a.HLS_L30.stacked.tif 129a.HLS_L30.2018-08-07.fmask.tif\n", - "Already chipped event 129a: skipping...\n", "Chipping: 130a 130a.MASK.tif 130a.HLS_L30.stacked.tif 130a.HLS_L30.2018-08-07.fmask.tif\n", - "Already chipped event 130a: skipping...\n", + "Chipping: 132a 132a.MASK.tif 132a.HLS_L30.stacked.tif 132a.HLS_L30.2018-08-11.fmask.tif\n", "Chipping: 133a 133a.MASK.tif 133a.HLS_L30.stacked.tif 133a.HLS_L30.2018-07-26.fmask.tif\n", - "Chipping: 613b 613b.MASK.tif 613b.HLS_L30.stacked.tif 613b.HLS_L30.2018-07-08.fmask.tif\n", + "Chipping: 614a 614a.MASK.tif 614a.HLS_L30.stacked.tif 614a.HLS_L30.2018-07-08.fmask.tif\n", "Chipping: 623d 623d.MASK.tif 623d.HLS_L30.stacked.tif 623d.HLS_L30.2019-08-19.fmask.tif\n", - "Already chipped event 623d: skipping...\n", "Chipping: 623a 623a.MASK.tif 623a.HLS_L30.stacked.tif 623a.HLS_L30.2019-08-19.fmask.tif\n", - "Already chipped event 623a: skipping...\n", - "Chipping: 889a 889a.MASK.tif 889a.HLS_L30.stacked.tif 889a.HLS_L30.2018-07-13.fmask.tif\n", - "Already chipped event 889a: skipping...\n", "Chipping: 915a 915a.MASK.tif 915a.HLS_L30.stacked.tif 915a.HLS_L30.2017-07-26.fmask.tif\n", "Chipping: 1378a 1378a.MASK.tif 1378a.HLS_L30.stacked.tif 1378a.HLS_L30.2018-07-12.fmask.tif\n", - "Chipping: 628a 628a.MASK.tif 628a.HLS_L30.stacked.tif 628a.HLS_L30.2019-07-18.fmask.tif\n", - "Already chipped event 628a: skipping...\n", - "Chipping: 116a 116a.MASK.tif 116a.HLS_L30.stacked.tif 116a.HLS_L30.2020-06-17.fmask.tif\n", - "Already chipped event 116a: skipping...\n", "Chipping: 1052a 1052a.MASK.tif 1052a.HLS_L30.stacked.tif 1052a.HLS_L30.2017-08-13.fmask.tif\n", - "Chipping: 1338a 1338a.MASK.tif 1338a.HLS_L30.stacked.tif 1338a.HLS_L30.2019-09-05.fmask.tif\n", - "Already chipped event 1338a: skipping...\n", - "Chipping: 1344a 1344a.MASK.tif 1344a.HLS_L30.stacked.tif 1344a.HLS_L30.2018-08-02.fmask.tif\n", "Chipping: 1376a 1376a.MASK.tif 1376a.HLS_L30.stacked.tif 1376a.HLS_L30.2020-07-19.fmask.tif\n" ] } ], "source": [ "chipped_events = set() \n", - "filtered_chips = []\n", - "all_chips = []\n", + "event_chip_stacks = []\n", "\n", "for evt, modality, (m, b, f) in event_files:\n", " if evt.name not in keepers[modality['id']]:\n", @@ -1804,56 +1553,205 @@ " data_chips = chip_data.chip_data(grid, b, chip_paths['all']['hls'])\n", " fmask_chips = chip_data.chip_data(grid, f, chip_paths['all']['other'])\n", "\n", - " stacks = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, modality)\n", - " all_chips += stacks\n", + " event_chips = chip_data.make_chip_stacks(data_chips, fmask_chips, label_chips, modality)\n", + " event_chip_stacks.append((event_chips, evt))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a022ca2c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "([ChipStack(id='000.000', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.000.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.000.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.000.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.001', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.001.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.001.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.001.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.002', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.002.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.002.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.002.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.003', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.003.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.003.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.003.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.004', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.004.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.004.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.004.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.005.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.005.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.006', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.006.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.006.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.006.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.007', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.007.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.007.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.007.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.008', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.008.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.008.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.008.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='000.009', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/000.009.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/000.009.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/000.009.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.000', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.000.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.000.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.000.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.001', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.001.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.001.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.001.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.002', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.002.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.002.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.002.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.003', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.003.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.003.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.003.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.004', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.004.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.004.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.004.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.005.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.005.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.006', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.006.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.006.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.006.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.007', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.007.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.007.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.007.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.008', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.008.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.008.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.008.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='001.009', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/001.009.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/001.009.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/001.009.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.000', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.000.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.000.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.000.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.001', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.001.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.001.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.001.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.002', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.002.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.002.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.002.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.003', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.003.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.003.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.003.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.004', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.004.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.004.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.004.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.005', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.005.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.005.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.005.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.006', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.006.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.006.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.006.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.007', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.007.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.007.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.007.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.008', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.008.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.008.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.008.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))}), ChipStack(id='002.009', data=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/HLS/002.009.102a.HLS_S30.stacked.tif'), validation_mask=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/OTHER/002.009.102a.HLS_S30.2019-07-17.Fmask.tif'), label=PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/chips/LABEL/002.009.102a.MASK.tif'), modality={'id': 'HLS_S30', 'collection': 'HLSS30', 'bands': (Band(id='B02', name='Blue', shortname='B'), Band(id='B03', name='Green', shortname='G'), Band(id='B04', name='Red', shortname='R'), Band(id='B8A', name='NIR Narrow', shortname='N'), Band(id='B11', name='SWIR 1', shortname='SW1'), Band(id='B12', name='SWIR 2', shortname='SW2'), Band(id='Fmask', name='Cloud Mask', shortname='Fmask'))})], Event(name='102a', date=Timestamp('2019-07-17 00:00:00'), wgs84_geometry=, buffer_m=10000))\n" + ] + } + ], + "source": [ + "print(event_chip_stacks[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "637de3af", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "filtering chips for 102a\n", + " damage: 10, no damage: 15\n", + "filtering chips for 126a\n", + " damage: 3, no damage: 4\n", + "filtering chips for 127a\n", + " damage: 16, no damage: 24\n", + "filtering chips for 134a\n", + " damage: 8, no damage: 12\n", + "filtering chips for 598a\n", + " damage: 7, no damage: 10\n", + "filtering chips for 889a\n", + " damage: 10, no damage: 15\n", + "filtering chips for 889b\n", + " damage: 5, no damage: 7\n", + "filtering chips for 892a\n", + " damage: 15, no damage: 22\n", + "filtering chips for 1079a\n", + " damage: 25, no damage: 37\n", + "filtering chips for 1069a\n", + " damage: 3, no damage: 4\n", + "filtering chips for 628a\n", + " damage: 1, no damage: 1\n", + "filtering chips for 638a\n", + " damage: 47, no damage: 70\n", + "filtering chips for 116d\n", + " damage: 6, no damage: 9\n", + "filtering chips for 116e\n", + " damage: 3, no damage: 4\n", + "filtering chips for 648a\n", + " damage: 4, no damage: 6\n", + "filtering chips for 1055a\n", + " damage: 24, no damage: 36\n", + "filtering chips for 1056a\n", + " damage: 22, no damage: 33\n", + "filtering chips for 1347a\n", + " damage: 18, no damage: 27\n", + "filtering chips for 1064a\n", + " damage: 13, no damage: 19\n", + "filtering chips for 1338a\n", + " damage: 7, no damage: 10\n", + "filtering chips for 1373a\n", + " damage: 3, no damage: 4\n", + "filtering chips for 1373d\n", + " damage: 4, no damage: 6\n", + "filtering chips for 1373e\n", + " damage: 7, no damage: 10\n", + "filtering chips for 110d\n", + " damage: 7, no damage: 10\n", + "filtering chips for 106a\n", + " damage: 23, no damage: 33\n", + "filtering chips for 129a\n", + " damage: 13, no damage: 19\n", + "filtering chips for 130a\n", + " damage: 17, no damage: 25\n", + "filtering chips for 132a\n", + " damage: 17, no damage: 25\n", + "filtering chips for 133a\n", + " damage: 3, no damage: 3\n", + "filtering chips for 614a\n", + " damage: 56, no damage: 84\n", + "filtering chips for 623d\n", + " damage: 8, no damage: 11\n", + "filtering chips for 623a\n", + " damage: 5, no damage: 7\n", + "filtering chips for 915a\n", + " damage: 7, no damage: 10\n", + "filtering chips for 1378a\n", + " damage: 4, no damage: 6\n", + "filtering chips for 1052a\n", + " damage: 4, no damage: 4\n", + "filtering chips for 1376a\n", + " damage: 1, no damage: 1\n" + ] + } + ], + "source": [ + "import random\n", + "\n", + "all_chips = []\n", + "filtered_chips = []\n", + "all_no_damage = []\n", + "all_damage = []\n", + "\n", + "for event_chips, event in event_chip_stacks:\n", + " print(f'filtering chips for {event.name}')\n", + " all_chips += event_chips\n", " view.view_chips(\n", - " stacks, \n", + " event_chips, \n", " modality, \n", " rgb_bands=[2, 1, 0], \n", - " save_to_file=chip_paths['all']['plots'] / f'{evt.name}.chips.png', \n", + " save_to_file=chip_paths['all']['plots'] / f'{event.name}.chips.png', \n", " quite=True\n", " )\n", " \n", - " filtered = chip_data.filter_chips(stacks)\n", + " damage_chips = chip_data.filter_damage_chips(event_chips)\n", + " all_damage += damage_chips\n", "\n", - " if len(filtered) == 0:\n", - " print(f'Filtered out all chips for {evt.name}')\n", + " no_damage_chips = chip_data.filter_no_damage_chips(event_chips)\n", + "\n", + " random.shuffle(no_damage_chips)\n", + " no_damage_chips = no_damage_chips[: int(len(damage_chips) * 1.5)]\n", + "\n", + " print(f' damage: {len(damage_chips)}, no damage: {len(no_damage_chips)}')\n", + "\n", + " if len(damage_chips) == 0 and len(no_damage_chips) == 0:\n", + " print(f'Filtered out all chips for {event.name}')\n", " continue\n", "\n", - " filtered_chips += filtered\n", + " all_no_damage += no_damage_chips\n", + " filtered = damage_chips + no_damage_chips\n", + " \n", " view.view_chips(\n", " filtered, \n", " modality, \n", " rgb_bands=[2, 1, 0], \n", - " save_to_file=chip_paths['output']['plots'] / f'{evt.name}.filtered.png', \n", + " save_to_file=chip_paths['output']['plots'] / f'{event.name}.filtered.png', \n", " quite=True\n", - " )" + " )\n", + " \n", + " filtered_chips += filtered" ] }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 21, "id": "252dff47", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(1, 1544)" + "(426, 623, 1049, 1688)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(all_damage), len(all_no_damage), len(filtered_chips), len(all_chips)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "7d4aeeb6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/output/HLS')" ] }, - "execution_count": 61, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "len(filtered), len(all_chips)" + "chip_paths['output']['hls']" ] }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 23, "id": "6a7dca8f", "metadata": {}, "outputs": [], @@ -1871,17 +1769,17 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 24, "id": "b1c32b82", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "392" + "1049" ] }, - "execution_count": 63, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1893,18 +1791,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "6f82016d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "86 308\n" - ] - } - ], + "outputs": [], "source": [ "#len(filtered_chips)\n", "#filtered_chips[0]\n", @@ -1917,7 +1807,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 26, "id": "3049d797", "metadata": {}, "outputs": [], @@ -1961,7 +1851,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 27, "id": "00880cff", "metadata": {}, "outputs": [ @@ -1976,7 +1866,7 @@ " PosixPath('/home/wbhorn/Repositories/fm/satchip/notebooks/data/HLS_L30/merged/1376a.HLS_L30.2020-07-19.SW2.tif')]" ] }, - "execution_count": 65, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1987,7 +1877,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 28, "id": "f68fa713", "metadata": {}, "outputs": [], @@ -1997,17 +1887,28 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 29, "id": "7a6d8b65", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "415" + "{'HLS': {'means': {'B': 267.3031005859375,\n", + " 'G': 500.4424133300781,\n", + " 'R': 332.49700927734375,\n", + " 'N': 4312.5751953125,\n", + " 'SW1': 1867.71044921875,\n", + " 'SW2': 887.797607421875},\n", + " 'stds': {'B': 112.45736694335938,\n", + " 'G': 160.5001678466797,\n", + " 'R': 199.83193969726562,\n", + " 'N': 594.2197265625,\n", + " 'SW1': 496.0038757324219,\n", + " 'SW2': 382.4466857910156}}}" ] }, - "execution_count": 55, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -2032,8 +1933,17 @@ "\n", "stats_file\n", "\n", - "(output_base / 'statistics.json').write_text(json.dumps(stats_file, indent=2))" + "(output_base / 'statistics.json').write_text(json.dumps(stats_file, indent=2))\n", + "stats_file" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "312e53f0", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/satchip/chip_data.py b/src/satchip/chip_data.py index 03ce27b..3d43657 100644 --- a/src/satchip/chip_data.py +++ b/src/satchip/chip_data.py @@ -90,14 +90,14 @@ def make_chip_stacks( return chip_stacks -def filter_chips(chip_stacks: list[models.ChipStack]) -> list[models.ChipStack]: +def filter_damage_chips(chip_stacks: list[models.ChipStack]) -> list[models.ChipStack]: good_chips = [] for chip_stack in chip_stacks: if 'HLS' in chip_stack.modality['id']: - is_good_chip = is_good_hls_chip(chip_stack) + is_good_chip = is_good_damage_hls_chip(chip_stack) elif 'RTC' in chip_stack.modality['id']: - is_good_chip = is_good_rtc_chip(chip_stack) + is_good_chip = is_good_damage_rtc_chip(chip_stack) if is_good_chip: good_chips.append(chip_stack) @@ -105,10 +105,24 @@ def filter_chips(chip_stacks: list[models.ChipStack]) -> list[models.ChipStack]: return good_chips -def is_good_hls_chip(chip_stack: models.ChipStack) -> bool: +def filter_no_damage_chips(chip_stacks: list[models.ChipStack]) -> list[models.ChipStack]: + good_chips = [] + + for chip_stack in chip_stacks: + if 'HLS' in chip_stack.modality['id']: + is_good_chip = is_good_no_damage_hls_chip(chip_stack) + elif 'RTC' in chip_stack.modality['id']: + is_good_chip = is_good_no_damage_rtc_chip(chip_stack) + + if is_good_chip: + good_chips.append(chip_stack) + + return good_chips + + +def _hls_chip_stats(chip_stack: models.ChipStack) -> tuple[float, float]: with rasterio.open(chip_stack.validation_mask) as ds: qc = clear_px_Fmask(ds.read(1)) - with rasterio.open(chip_stack.label) as ds: event = ds.read(1) @@ -119,74 +133,58 @@ def is_good_hls_chip(chip_stack: models.ChipStack) -> bool: n_cf = len(np.where(qc == 0)[0]) pct_cf = 100.0 * (n_cf / n_px) - # event pixels - n_ev = len(np.where(event > 0)[0]) - # pct of chip in event + n_ev = len(np.where(event > 0)[0]) pct_ev = 100.0 * (n_ev / n_px) if n_ev > 0 else 0 + return pct_cf, pct_ev + + +def is_good_damage_hls_chip(chip_stack: models.ChipStack) -> bool: + pct_cf, pct_ev = _hls_chip_stats(chip_stack) return pct_cf > 95 and pct_ev > 1 +def is_good_no_damage_hls_chip(chip_stack: models.ChipStack) -> bool: + pct_cf, pct_ev = _hls_chip_stats(chip_stack) + return pct_cf > 95 and pct_ev < 1 + + def clear_px_Fmask(Fmask: np.ndarray) -> np.ndarray: fmask_clear = np.array( - [ - 0, - 4, - 16, - 20, - 32, - 36, - 48, - 52, - 64, - 68, - 80, - 84, - 96, - 100, - 112, - 116, - 128, - 132, - 144, - 148, - 160, - 164, - 176, - 180, - 192, - 196, - 208, - 212, - 224, - 228, - 240, - 244, - ], + [0, 4, 16, 20, 32, 36, 48, 52, 64, 68, 80, 84, 96, 100, 112, 116, + 128, 132, 144, 148, 160, 164, 176, 180, 192, 196, 208, 212, 224, 228, 240, 244], dtype=Fmask.dtype, ) - + # 0 clear, 1 cloud, 255 nodata cloudmask = np.ones_like(Fmask, dtype=np.uint8) cloudmask[np.isin(Fmask, fmask_clear)] = 0 cloudmask[Fmask == 255] = 255 - return cloudmask -def is_good_rtc_chip(chip_stack: models.ChipStack) -> bool: +def _rtc_chip_stats(chip_stack: models.ChipStack) -> tuple[bool, bool]: with rasterio.open(chip_stack.data) as ds: rtc_data = ds.read() - with rasterio.open(chip_stack.label) as ds: event_mask = ds.read(1) has_nan_pixels = np.isnan(rtc_data).sum() > 0 + # pct of chip in event num_pixels = event_mask.size num_event_pixels = np.count_nonzero(event_mask > 0) - pct_pixels_over_event = 100.0 * (num_event_pixels / num_pixels) data_overlaps_event = pct_pixels_over_event > 1 + return has_nan_pixels, data_overlaps_event + + +def is_good_damage_rtc_chip(chip_stack: models.ChipStack) -> bool: + has_nan_pixels, data_overlaps_event = _rtc_chip_stats(chip_stack) return not has_nan_pixels and data_overlaps_event + + +def is_good_no_damage_rtc_chip(chip_stack: models.ChipStack) -> bool: + has_nan_pixels, data_overlaps_event = _rtc_chip_stats(chip_stack) + return not has_nan_pixels and not data_overlaps_event