From 4d263ebefed74bc060482868edd48e3c3751d4d8 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 13:16:31 +0200
Subject: [PATCH 01/61] add ApsParameters to objects.py

---
 sarvey/objects.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/sarvey/objects.py b/sarvey/objects.py
index accac3db..06bd00d2 100644
--- a/sarvey/objects.py
+++ b/sarvey/objects.py
@@ -793,3 +793,89 @@ def open(self, *, path_inputs: str):
             self.demerr = f["demerr"][:]
             self.vel = f["vel"][:]
             self.gamma = f["gamma"][:]
+
+
+class ApsParameters:
+    """APS model parameters."""
+    def __init__(self, *, file_path: str, logger: Logger):
+        """Init.
+
+        Parameters
+        ----------
+        file_path: str
+            path to filename
+        logger: Logger
+            Logging handler
+        """
+        self.file_path = file_path
+        self.model_name = None
+        self.model_params = None
+        self.num_params = None
+        self.phase = None
+        self.logger = logger
+        self.logger.debug(f"ApsParameters initialized with file path: {file_path}")
+
+    def prepare(self, *, model_name: str = "Stable", model_params: np.ndarray, phase: np.ndarray):
+        """prepare APS model parameters and store it into a file.
+
+        Parameters
+        ----------
+        model_name: str
+            name of the model
+        model_params: np.ndarray
+            Model parameters
+        phase: np.ndarray
+            Residual phase of P1 used for APS estimation
+        """
+        self.logger.info(f"Preparing ApsParameters with model: {model_name}")
+
+        if model_params is None or model_params.size == 0:
+            self.logger.debug("Model parameters are empty; defaulting to an empty list.")
+            model_params = []
+
+        self.model_name = model_name
+        self.model_params = model_params
+        self.num_params = model_params.shape[0] if model_params is not None and model_params.size > 0 else 0
+        self.phase = phase
+
+        self.logger.debug(f"Model parameters set with {self.num_params} parameters.")
+        self.logger.debug(f"Phase data shape: {phase.shape}")
+
+        self.writeToFile()
+
+    def open(self):
+        """Read data from file.
+
+        Read stored information from already existing .h5 file.
+        """
+
+        self.logger.info(f"Opening file: {self.file_path}")
+
+        try:
+            with h5py.File(self.file_path, 'r') as f:
+                self.model_params = f["model_params"][:]
+                self.phase = f["phase"][:]
+                self.model_name = f.attrs["model_name"]
+                self.num_params = f.attrs["num_params"]
+                self.logger.debug(f"Loaded model name: {self.model_name}, number of parameters: {self.num_params}")
+        except Exception as e:
+            self.logger.error(f"Failed to open file {self.file_path}: {e}")
+
+    def writeToFile(self):
+        """Write data to .h5 file."""
+        self.logger.info(msg="write data to {}...".format(self.file_path))
+
+        try:
+            if exists(self.file_path):
+                self.logger.warning(f"File {self.file_path} exists and will be overwritten.")
+                os.remove(self.file_path)
+
+            with h5py.File(self.file_path, 'w') as f:
+                f.create_dataset('model_params', data=self.model_params)
+                f.create_dataset('phase', data=self.phase)
+                f.attrs["model_name"] = self.model_name
+                f.attrs["num_params"] = self.num_params
+
+            self.logger.info(f"Data successfully written to {self.file_path}.")
+        except Exception as e:
+            self.logger.error(f"Error writing data to {self.file_path}: {e}")

From d35a302f7d31b80bfed4d78dc6d7a27467a67d53 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 13:26:14 +0200
Subject: [PATCH 02/61] fix lint

---
 sarvey/objects.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sarvey/objects.py b/sarvey/objects.py
index 06bd00d2..1def7876 100644
--- a/sarvey/objects.py
+++ b/sarvey/objects.py
@@ -797,6 +797,7 @@ def open(self, *, path_inputs: str):
 
 class ApsParameters:
     """APS model parameters."""
+
     def __init__(self, *, file_path: str, logger: Logger):
         """Init.
 
@@ -816,7 +817,7 @@ def __init__(self, *, file_path: str, logger: Logger):
         self.logger.debug(f"ApsParameters initialized with file path: {file_path}")
 
     def prepare(self, *, model_name: str = "Stable", model_params: np.ndarray, phase: np.ndarray):
-        """prepare APS model parameters and store it into a file.
+        """Prepare APS model parameters and store it into a file.
 
         Parameters
         ----------
@@ -848,7 +849,6 @@ def open(self):
 
         Read stored information from already existing .h5 file.
         """
-
         self.logger.info(f"Opening file: {self.file_path}")
 
         try:

From 3a36c6ef509008304513ec152d83aa64c7dc0e91 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 18:26:40 +0200
Subject: [PATCH 03/61] fix bug in ApsParameters when model_params is None

---
 sarvey/objects.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sarvey/objects.py b/sarvey/objects.py
index 1def7876..fe732db7 100644
--- a/sarvey/objects.py
+++ b/sarvey/objects.py
@@ -832,7 +832,7 @@ def prepare(self, *, model_name: str = "Stable", model_params: np.ndarray, phase
 
         if model_params is None or model_params.size == 0:
             self.logger.debug("Model parameters are empty; defaulting to an empty list.")
-            model_params = []
+            model_params = np.array([])
 
         self.model_name = model_name
         self.model_params = model_params

From 8028550112e845d3ad39610b8bbe12a56dab58a7 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 19:32:07 +0200
Subject: [PATCH 04/61] implement extractModelParams and applyModelParams in
 filtering.

---
 sarvey/filtering.py | 69 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 2ca250ed..4841ea1a 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -289,3 +289,72 @@ def simpleInterpolation(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_
         )
 
     return aps1, aps2
+
+
+def extractModelParams(model: gs.covmodel.models, logger: Logger):
+    """
+    Extract parameters from the given gstools model.
+
+    Parameters
+    ----------
+    model : gs.covmodel.models
+        The model from which to extract parameters.
+    logger : Logger
+        Logging handler.
+
+    Returns
+    -------
+    tuple
+        A tuple containing the model parameters.
+
+    Raises
+    ------
+    ValueError
+        If the model is not implemented.
+    """
+    if model.name == 'Stable':
+        params = (model.var, model.len_scale, model.nugget, model.alpha)
+        logger.debug(msg=f"Extract {params.size} parameters from gs model {model.name}.")
+    # elif model.name == 'Gaussian':
+    #     params = (model.var, model.len_scale, model.nugget)
+    else:
+        error_msg = f"Model {model.name} not implemented yet."
+        logger.error(msg=error_msg)
+        raise ValueError(error_msg)
+    return params
+
+
+def applyModelParams(model: gs.covmodel.models, params: tuple, logger: Logger):
+    """
+    Apply parameters to the given gstools model.
+
+    Parameters
+    ----------
+    model : gs.covmodel.models
+        The model to which parameters will be applied.
+    params : tuple
+        A tuple containing the model parameters.
+    logger : Logger
+        Logging handler.
+
+    Returns
+    -------
+    model
+        The model with applied parameters.
+
+    Raises
+    ------
+    ValueError
+        If the model is not implemented.
+    """
+
+    if model.name == 'Stable':
+        logger.debug(msg=f"Applying {params.size} parameters to gs model {model.name}.")
+        model.var, model.len_scale, model.nugget, model.alpha = params
+    # elif model.name == 'Gaussian':  # TODO: implement other models
+    #     model.var, model.len_scale, model.nugget = params
+    else:
+        error_msg = f"Model {model.name} not implemented yet."
+        logger.error(msg=error_msg)
+        raise ValueError(error_msg)
+    return model

From 7d084a17a5f24dd0275d8da502ab79c7f30b41fe Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 21:22:00 +0200
Subject: [PATCH 05/61] fix bug in params size check in extractModelParams

---
 sarvey/filtering.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 4841ea1a..9fe7b091 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -314,7 +314,7 @@ def extractModelParams(model: gs.covmodel.models, logger: Logger):
     """
     if model.name == 'Stable':
         params = (model.var, model.len_scale, model.nugget, model.alpha)
-        logger.debug(msg=f"Extract {params.size} parameters from gs model {model.name}.")
+        logger.debug(msg=f"Extract {len(params)} parameters from gs model {model.name}.")
     # elif model.name == 'Gaussian':
     #     params = (model.var, model.len_scale, model.nugget)
     else:

From 867cb1265e8ff029787506472583f3f7cfbeb3eb Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 21:51:31 +0200
Subject: [PATCH 06/61] remove logger from applyModelParams to avoid
 overlogging in loops.

---
 sarvey/filtering.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 9fe7b091..8a1b1454 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -349,7 +349,6 @@ def applyModelParams(model: gs.covmodel.models, params: tuple, logger: Logger):
     """
 
     if model.name == 'Stable':
-        logger.debug(msg=f"Applying {params.size} parameters to gs model {model.name}.")
         model.var, model.len_scale, model.nugget, model.alpha = params
     # elif model.name == 'Gaussian':  # TODO: implement other models
     #     model.var, model.len_scale, model.nugget = params

From 654756cd4d533a984443da3d585876c5d300dac0 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 23:44:02 +0200
Subject: [PATCH 07/61] implement applySpatialFilteringToP2

---
 sarvey/filtering.py | 67 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 8a1b1454..618dfd74 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -41,6 +41,73 @@
 import sarvey.utils as ut
 
 
+def applySpatialFilteringToP2(*, coord_utm1, residuals, coord_utm2, model_name, model_params, logger):
+    """
+    Apply spatial filtering to second-order points using kriging.
+
+    Parameters
+    ----------
+    coord_utm1 : np.ndarray
+        Coordinates of the first-order points in UTM.
+    residuals : np.ndarray
+        Residuals of the first-order points that were used to estimate APS parameters.
+    coord_utm2 : np.ndarray
+        Coordinates of the second-order points in UTM.
+    model_name : str
+        Name of the variogram model used in APS estimation.
+    model_params : np.ndarray
+        Parameters of variogram model used in APS estimation.
+    logger : Logger
+        Logger instance for logging messages.
+
+    Returns
+    -------
+    aps2 : np.ndarray
+        Interpolated phase values for the second-order points.
+    """
+    num_time = model_params.shape[1]
+    num_points2 = coord_utm2.shape[0]
+    logger.debug(f"Starting applying {num_time} time APS to {num_points2} second-order points using Kriging"
+                 f" interpolation.")
+
+    if model_name == 'Stable':
+        logger.debug(f"Recreate {model_name} gs model using {model_params.shape[0]} parameters.")
+        model = gs.Stable(dim=2)
+        if model_params.shape[0] != 4:
+            error_msg = (f"Number of model parameters does not match the number of parameters required by "
+                         f"{model_name} model")
+            logger.error(msg=error_msg)
+            raise ValueError(error_msg)
+    else:
+        error_msg = f"Model {model_name} not implemented yet."
+        logger.warning(msg=error_msg)
+        raise ValueError(error_msg)
+
+    coord_point1 = [coord_utm1[:, 0], coord_utm1[:, 1]]
+
+    aps2 = np.zeros((num_points2, num_time), dtype=np.float32)
+
+    logger.debug(msg=f"Start looping for kriging parameters estimation and APS interpolation for {num_time} snapshots.")
+    for i in range(num_time):
+        try:
+            model = applyModelParams(model, model_params[:, i], logger)
+        except Exception as e:
+            warning_msg = f"Model parameters for time step {i} are not provided: {e}"
+            logger.warning(msg=warning_msg)
+            raise ValueError(warning_msg)
+
+        field = residuals[:, i].astype(np.float32)
+
+        # estimate kriging parameters based on the variogram model and the residuals of first order points
+        sk = gs.krige.Simple(model=model, cond_pos=coord_point1, cond_val=field)
+
+        # evaluate the kriging model at new locations
+        aps2[:, i], var_sk_new = sk((coord_utm2[:, 1], coord_utm2[:, 0]), return_var=True)
+
+    logger.debug(msg="Finished applying spatial filtering to second-order points using Kriging interpolation.")
+    return aps2
+
+
 def launchSpatialFiltering(parameters: tuple):
     """Launch_spatial_filtering.
 

From b0a508cfdd5772a3434afb3e918c38d918b65fdf Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 23:52:38 +0200
Subject: [PATCH 08/61] small debug message change

---
 sarvey/filtering.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 618dfd74..d3722df5 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -67,7 +67,7 @@ def applySpatialFilteringToP2(*, coord_utm1, residuals, coord_utm2, model_name,
     """
     num_time = model_params.shape[1]
     num_points2 = coord_utm2.shape[0]
-    logger.debug(f"Starting applying {num_time} time APS to {num_points2} second-order points using Kriging"
+    logger.debug(f"Start applying {num_time} time APS to {num_points2} second-order points using Kriging"
                  f" interpolation.")
 
     if model_name == 'Stable':

From d57e3d34b8f171eab538331253fe10a2a3cf35ef Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 2 Aug 2024 23:59:46 +0200
Subject: [PATCH 09/61] implement applySimpleInterpolationToP2

---
 sarvey/filtering.py | 49 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index d3722df5..f8e5253e 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -108,6 +108,55 @@ def applySpatialFilteringToP2(*, coord_utm1, residuals, coord_utm2, model_name,
     return aps2
 
 
+def applySimpleInterpolationToP2(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_utm2: np.ndarray,
+                                 logger: Logger, interp_method: str = "linear",):
+    """
+    Apply simple interpolation to second-order points.
+
+    Parameters
+    ----------
+    logger
+    residuals : np.ndarray
+        Residual phase values from the first-order points (size: num_points_p1 x num_ifgs)
+    coord_utm1 : np.ndarray
+        coordinates in UTM of the points for which the residuals are given (size: num_points_p1 x 2)
+    coord_utm2 : np.ndarray
+        Coordinates of the second-order points in UTM. (size: num_points_p2 x 2)
+    logger: Logger
+        Logger instance for logging messages.
+
+    interp_method : str
+        Interpolation method (default: "linear"; options: "linear", "cubic")
+
+    Returns
+    -------
+    interpolated_phase : np.ndarray
+        Atmospheric phase screen for the second order points (size: num_points_p2 x num_images)
+    """
+    num_points2 = coord_utm2.shape[0]
+    num_images = residuals.shape[1]
+    logger.debug(f"Start applying {num_images} APS to {num_points2} second-order points using"
+                 f" {interp_method} interpolation.")
+
+    aps2 = np.zeros((num_points2, num_images), dtype=np.float32)
+
+    logger.debug(msg=f"Start looping for APS interpolation for {num_images} snapshots.")
+    for i in range(num_images):
+        aps2[:, i] = griddata(coord_utm1, residuals[:, i], coord_utm2, method=interp_method)
+        # interpolation with 'linear' or 'cubic' yields nan values for pixel that need to be extrapolated.
+        # interpolation with 'knn' solves this problem.
+        mask_extrapolate = np.isnan(aps2[:, i])
+        aps2[mask_extrapolate, i] = griddata(
+            coord_utm1,
+            residuals[:, i],
+            coord_utm2[mask_extrapolate, :],
+            method='nearest'
+        )
+
+    logger.debug(msg=f"Finished applying spatial filtering to second-order points using {interp_method} interpolation.")
+    return aps2
+
+
 def launchSpatialFiltering(parameters: tuple):
     """Launch_spatial_filtering.
 

From 2199db3b572e5a866cbe76799c1633b13be06697 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sat, 3 Aug 2024 01:09:11 +0200
Subject: [PATCH 10/61] implement selectP2 in densification

---
 sarvey/densification.py | 242 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 240 insertions(+), 2 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 0d9ed911..73b7f4d8 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -28,17 +28,22 @@
 # with this program. If not, see <https://www.gnu.org/licenses/>.
 
 """Densification module for SARvey."""
+from os.path import join
 import time
 import multiprocessing
 import numpy as np
 from scipy.spatial import KDTree
 from logging import Logger
+import matplotlib.pyplot as plt
 
-from mintpy.utils import ptime
+from mintpy.utils import ptime, readfile
+from miaplpy.objects.slcStack import slcStack
 
 from sarvey.unwrapping import oneDimSearchTemporalCoherence
-from sarvey.objects import Points
+from sarvey.objects import Points, AmplitudeImage, BaseStack, ApsParameters
 import sarvey.utils as ut
+from sarvey.preparation import selectPixels, createTimeMaskFromDates
+from sarvey.filtering import applySimpleInterpolationToP2, applySpatialFilteringToP2
 
 
 def densificationInitializer(tree_p1: KDTree, tree_p2: KDTree, point2_obj: Points, demod_phase1: np.ndarray):
@@ -322,3 +327,236 @@ def densifyNetwork(*, point1_obj: Points, vel_p1: np.ndarray, demerr_p1: np.ndar
     gamma_p2 = gamma_p2[sort_idx]
     mean_gamma = mean_gamma[sort_idx]
     return demerr_p2, vel_p2, gamma_p2, mean_gamma
+
+
+def selectP2(*, output_path: str, config: dict, logger: Logger):
+    """Select second order points and interpolate APS to them
+
+    Parameters
+    ----------
+    output_path : str
+        path to the directory with processing files.
+    config : dict
+        dictionay containing parameters.
+    logger : Logger
+        Logger instance for logging messages.
+
+    Returns:
+    -------
+    point2_obj : Points
+        Points object with second-order points
+    aps2_obj: Points
+        Points object with APS estimation for second-order points
+
+    """
+    # this function is mainly adapted from sarvey.processing.runFiltering
+    # TODO: Directly pass input parameters instead of config dictionary
+    coherence_p2 = config.filtering.coherence_p2
+    logger.debug(f"Start Selecting 2nd order points with coherence threshold {coherence_p2:.2f}.")
+
+    coh_value = int(coherence_p2 * 100)
+
+    p1_file = join(output_path, "p1_ts_filt.h5")
+    logger.debug(f"Create a mask based on points in file {p1_file}.")
+    point1_obj = Points(file_path=p1_file, logger=logger)
+    point1_obj.open(path_inputs=config.data_directories.path_inputs)
+    p1_mask = point1_obj.createMask()  # used later for selecting psCand2 when a spatial mask AOI is given.
+    logger.debug(f"Mask created with {p1_mask.sum()} selected points.")
+
+    bmap_file = join(output_path, "background_map.h5")
+    logger.debug(f"Use file {bmap_file} to create a mask invalid points.")
+    bmap_obj = AmplitudeImage(file_path=bmap_file)
+
+    point_id_img = np.arange(0, point1_obj.length * point1_obj.width).reshape(
+        (point1_obj.length, point1_obj.width))
+
+    logger.debug(f"Select second-order points based on temporal coherence {coherence_p2:.2f}.")
+    cand_mask2 = selectPixels(
+        path=output_path, selection_method="temp_coh",
+        thrsh=coherence_p2,
+        grid_size=None, bool_plot=True,
+        logger=logger
+    )  # first-order points are included in second-order points
+    logger.debug(f"{cand_mask2.sum()} second-order points selected based on temporal coherence {coherence_p2:.2f}.")
+
+    if config.phase_linking.phase_linking:
+        # read PL results
+        pl_file = join(config.phase_linking.path_inverted, "phase_series.h5")
+        logger.debug(f"Read phase linking results from file {pl_file}.")
+        pl_coh = readfile.read(pl_file,
+                               datasetName='temporalCoherence')[0]
+        pl_coh = pl_coh[1, :, :]
+        siblings = readfile.read(pl_file,
+                                 datasetName='shp')[0]
+
+        if config.phase_linking.use_ps:
+            logger.debug(f"Read phase linking PS mask from file {config.phase_linking.path_mask_file_ps}.")
+            mask_ps = readfile.read(config.phase_linking.path_mask_file_ps,
+                                    datasetName='mask')[0].astype(np.bool_)
+            cand_mask_pl = (pl_coh > coherence_p2) | mask_ps
+        else:
+            cand_mask_pl = (pl_coh > coherence_p2)
+            # remove ps candidates, because the ps detection strategy in miaplpy seems to be biased.
+            cand_mask_pl[siblings <= config.phase_linking.num_siblings] = False
+
+        if config.phase_linking.spatial_mask_file_pl is not None:
+            path_mask_pl_aoi = join(config.phase_linking.spatial_mask_file_pl)
+            logger.debug(f"Load mask for area of interest from file {path_mask_pl_aoi}.")
+            mask_pl_aoi = readfile.read(path_mask_pl_aoi, datasetName='mask')[0].astype(np.bool_)
+
+            fig = plt.figure(figsize=[15, 5])
+            ax = fig.add_subplot()
+            ax.imshow(mask_pl_aoi, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
+            bmap_obj.plot(ax=ax, logger=logger)
+            coord_xy = np.array(np.where(cand_mask_pl)).transpose()
+            val = np.ones_like(cand_mask_pl)
+            sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask_pl], s=0.5,
+                            cmap=plt.get_cmap("autumn_r"),
+                            vmin=1, vmax=2)  # set min, max to ensure that points are yellow
+            cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
+            cbar.ax.set_visible(False)  # make size of axis consistent with all others
+            plt.tight_layout()
+            plt.title("Mask for phase linking points")
+            fig.savefig(join(output_path, "pic", "step_3_mask_coh{}_phase_linking.png".format(coh_value)), dpi=300)
+            plt.close(fig)
+
+            # mask points after plotting, so that available coherent points are visible in figure
+            cand_mask_pl[~mask_pl_aoi] = False
+
+        # combine phase linking coherence with TPC cand_mask2
+        initial_num_cand_mask2 = cand_mask2.sum()
+        cand_mask2 = cand_mask2 | cand_mask_pl
+        logger.debug(f"{cand_mask2.sum()-initial_num_cand_mask2} additional pixels selected by phase linking.")
+
+    mask_valid_area = ut.detectValidAreas(bmap_obj=bmap_obj, logger=logger)
+
+    if config.filtering.spatial_mask_file_p2 is not None:
+        path_mask_aoi = join(config.filtering.spatial_mask_file_p2)
+        logger.info(f"Load mask for area of interest from file {path_mask_aoi}.")
+        mask_aoi = readfile.read(path_mask_aoi, datasetName='mask')[0].astype(np.bool_)
+        mask_valid_area &= mask_aoi
+        # todo: add unstable points from p1 for densification
+    else:
+        logger.info(msg="No mask for area of interest given.")
+
+    cand_mask2[p1_mask] = True  # add all selected 1.order points to avoid spatial gaps in 2D unwrapping
+    # cand_mask2[cand_mask_sparse] = True  # add only stable points from 1.order points
+
+    cand_mask2 &= mask_valid_area
+    logger.info(f"Final number of selected second-order points: {cand_mask2.sum()}.")
+
+    fig = plt.figure(figsize=[15, 5])
+    ax = fig.add_subplot()
+    ax.imshow(mask_valid_area, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
+    bmap_obj.plot(ax=ax, logger=logger)
+    coord_xy = np.array(np.where(cand_mask2)).transpose()
+    val = np.ones_like(cand_mask2)
+    sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask2], s=0.5, cmap=plt.get_cmap("autumn_r"),
+                    vmin=1, vmax=2)  # set min, max to ensure that points are yellow
+    cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
+    cbar.ax.set_visible(False)  # make size of axis consistent with all others
+    plt.tight_layout()
+    plt.title("Mask for dense point set")
+    fig.savefig(join(output_path, "pic", "step_3_mask_coh{}.png".format(coh_value)), dpi=300)
+    plt.close(fig)
+
+    p2_file = join(output_path, f"coh{coh_value}_ifg_wr.h5")
+    logger.debug(f"Prepare second-order points object to save to file {p2_file}.")
+    point2_obj = Points(file_path=p2_file, logger=logger)
+    coord_xy = np.array(np.where(cand_mask2)).transpose()
+    point_id2 = point_id_img[cand_mask2]
+    point2_obj.prepare(
+        point_id=point_id2,
+        coord_xy=coord_xy,
+        path_inputs=config.data_directories.path_inputs
+    )
+
+    ifg_stack_file = join(output_path, "ifg_stack.h5")
+    logger.debug(f"Read interferogram phase for selected second-order points from file {ifg_stack_file}.")
+    ifg_stack_obj = BaseStack(file=ifg_stack_file, logger=logger)
+    point2_obj.phase = ut.readPhasePatchwise(stack_obj=ifg_stack_obj, dataset_name="ifgs",
+                                             num_patches=config.processing.num_patches, cand_mask=cand_mask2,
+                                             point_id_img=point_id_img, logger=logger)
+
+    if config.phase_linking.phase_linking:
+        pl_inverted_file = join(config.phase_linking.path_inverted, "phase_series.h5")
+        logger.debug(f"Read interferogram phase from MiaplPy results from file {pl_inverted_file}.")
+        phase_linking_obj = BaseStack(file=pl_inverted_file,
+                                      logger=logger)
+
+        pl_phase = ut.readPhasePatchwise(
+            stack_obj=phase_linking_obj, dataset_name="phase",
+            num_patches=config.processing.num_patches,
+            cand_mask=cand_mask2,
+            point_id_img=point_id_img, logger=logger
+        )
+
+        # subset to time span
+        slc_stack_obj = slcStack(join(config.data_directories.path_inputs, "slcStack.h5"))
+        slc_stack_obj.open()
+        time_mask = createTimeMaskFromDates(
+            start_date=config.preparation.start_date,
+            stop_date=config.preparation.stop_date,
+            date_list=slc_stack_obj.dateList,
+            logger=logger
+        )[0]
+        pl_phase = pl_phase[:, time_mask]
+
+        pl_ifgs = np.zeros((point2_obj.num_points, point2_obj.ifg_net_obj.num_ifgs), dtype=np.float32)
+
+        c = 0
+        for i, j in np.asarray(point1_obj.ifg_net_obj.ifg_list):
+            pl_ifgs[:, c] = np.angle(np.exp(1j * pl_phase[:, i]) * np.conjugate(np.exp(1j * pl_phase[:, j])))
+            c += 1
+
+        # change only phase for good phase linking pixels and keep original phase for good tpc pixels
+        mask_pl = cand_mask_pl[cand_mask2]
+        point2_obj.phase[mask_pl] = pl_ifgs[mask_pl]
+
+    logger.debug(f"Write second-order points to file {point2_obj.file_path}.")
+    point2_obj.writeToFile()
+    del ifg_stack_obj
+
+    p1_aps_file = join(output_path, "p1_aps.h5")
+    logger.debug(f"Read APS for first order point from file {p1_aps_file}.")
+    aps1_obj = Points(file_path=p1_aps_file, logger=logger)
+    aps1_obj.open(path_inputs=config.data_directories.path_inputs)
+
+    aps_params_file = join(output_path, "aps_parameters.h5")
+    logger.debug(f"Read APS parameters from file {aps_params_file}.")
+    aps_params_obj = ApsParameters(file_path=aps_params_file, logger=logger)
+    aps_params_obj.open()
+
+    if config.filtering.skip_filtering:
+        logger.info("Skip APS filtering")
+        # TODO: check if this is correct ####
+        # original code: aps2_phase = np.zeros_like(point2_obj.phase)
+        aps2_phase = np.zeros((point2_obj.num_points, aps_params_obj.phase.shape[1]))
+    else:
+        logger.debug("Apply APS filtering to second-order points.")
+        if config.filtering.interpolation_method == "kriging":
+            aps2_phase = applySpatialFilteringToP2(coord_utm1=aps1_obj.coord_utm,
+                                                   residuals=aps_params_obj.phase,
+                                                   coord_utm2=point2_obj.coord_utm,
+                                                   model_name=aps_params_obj.model_name,
+                                                   model_params=aps_params_obj.model_params,
+                                                   logger=logger)
+        else:
+            aps2_phase = applySimpleInterpolationToP2(residuals=aps_params_obj.phase,
+                                                      coord_utm1=aps1_obj.coord_utm,
+                                                      coord_utm2=point2_obj.coord_utm,
+                                                      logger=logger,
+                                                      interp_method=config.filtering.interpolation_method)
+
+    p2_aps_file = join(output_path, f"coh{coh_value}_aps.h5")
+    logger.info(f"Write APS for second-order points to file {p2_aps_file}.")
+    aps2_obj = Points(file_path=p2_aps_file, logger=logger)
+    aps2_obj.open(
+        other_file_path=join(output_path, f"coh{coh_value}_ifg_wr.h5"),
+        path_inputs=config.data_directories.path_inputs
+    )
+    aps2_obj.phase = aps2_phase
+    aps2_obj.writeToFile()
+
+    logger.debug("Finished selecting second-order points.")
+    return point2_obj, aps2_obj

From dae8193a3ee387dce1cddb8133bc2c56bd0d77f7 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 14:06:36 +0200
Subject: [PATCH 11/61] update logger messaeg in selectP2

---
 sarvey/densification.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 73b7f4d8..0c94685f 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -352,7 +352,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     # this function is mainly adapted from sarvey.processing.runFiltering
     # TODO: Directly pass input parameters instead of config dictionary
     coherence_p2 = config.filtering.coherence_p2
-    logger.debug(f"Start Selecting 2nd order points with coherence threshold {coherence_p2:.2f}.")
+    logger.debug(f"Start Selecting 2nd order points.")
 
     coh_value = int(coherence_p2 * 100)
 

From 655fb835ef27b21f60b72e252305c45c3d302e90 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 14:13:46 +0200
Subject: [PATCH 12/61] save aps parameters in step3 and select p2 points and
 apply aps to them in step4.

---
 sarvey/filtering.py  | 101 ++++++------------------
 sarvey/processing.py | 184 ++++++-------------------------------------
 2 files changed, 50 insertions(+), 235 deletions(-)

diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index f8e5253e..bbdb90b8 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -175,8 +175,6 @@ def launchSpatialFiltering(parameters: tuple):
             residual phase (size: num_points x num_ifgs)
         coord_utm1: np.ndarray
             coordinates in UTM of the first-order points for which the residuals are given (size: num_points_p1 x 2)
-        coord_utm2: np.ndarray
-            coordinates in UTM of the new points which shall be interpolated (size: num_points_p2 x 2)
         bins: np.ndarray
             bin edges for the variogram
         bool_plot: bool
@@ -190,22 +188,27 @@ def launchSpatialFiltering(parameters: tuple):
         range of indices for the time series
     aps1: np.ndarray
         atmospheric phase screen for the known points (size: num_points_p1 x num_ifgs)
-    aps2: np.ndarray
-        atmospheric phase screen for the new points (size: num_points_p2 x num_ifgs)
     """
     # Unpack the parameters
-    (idx_range, num_time, residuals, coord_utm1, coord_utm2, bins, bool_plot, logger) = parameters
+    (idx_range, num_time, residuals, coord_utm1, bins, bool_plot, logger) = parameters
 
     x = coord_utm1[:, 1]
     y = coord_utm1[:, 0]
-    x_new = coord_utm2[:, 1]
-    y_new = coord_utm2[:, 0]
 
     aps1 = np.zeros((coord_utm1.shape[0], num_time), dtype=np.float32)
-    aps2 = np.zeros((coord_utm2.shape[0], num_time), dtype=np.float32)
 
     prog_bar = ptime.progressBar(maxValue=num_time)
 
+    model_name = 'Stable'
+    if model_name == 'Stable':
+        model = gs.Stable(dim=2)
+        params = extractModelParams(model, logger)
+        model_params = np.array([np.full(num_time, param) for param in params], dtype=np.float32)
+    else:  #  TODO: Other models to be added later
+        error_msg = f"Model {model_name} not implemented yet."
+        logger.error(msg=error_msg)
+        raise ValueError(error_msg)
+
     for i in range(num_time):
         field = residuals[:, i].astype(np.float32)
 
@@ -213,9 +216,9 @@ def launchSpatialFiltering(parameters: tuple):
         bin_center, vario = gs.vario_estimate(pos=[x, y], field=field, bin_edges=bins)
 
         # 2) fit model to empirical variogram
-        model = gs.Stable(dim=2)
         try:
             model.fit_variogram(x_data=bin_center, y_data=vario, nugget=True, max_eval=1500)
+            model_params[:, i] = extractModelParams(model=model, logger=logger)
         except RuntimeError as err:
             logger.error(msg="\nIMAGE {}: Not able to fit variogram! {}".format(idx_range[i], err))
             if bool_plot:
@@ -241,47 +244,12 @@ def launchSpatialFiltering(parameters: tuple):
         fld_sk, _ = sk((x, y), return_var=True)
         aps1[:, i] = fld_sk
 
-        # 5) evaluate the kriging model at NEW locations
-        fld_sk_new, var_sk_new = sk((x_new, y_new), return_var=True)
-        aps2[:, i] = fld_sk_new
-
         prog_bar.update(value=i + 1, every=1, suffix='{}/{} images'.format(i + 1, num_time))
 
-        # 5) show results
-        if bool_plot:
-            min_val = np.min(field)
-            max_val = np.max(field)
-
-            fig, ax = plt.subplots(2, 2, figsize=[10, 5])
-
-            cur_ax = ax[0, 0]
-            sca1 = cur_ax.scatter(x, y, c=field, vmin=min_val, vmax=max_val)
-            plt.colorbar(sca1, ax=cur_ax, pad=0.03, shrink=0.5)
-            cur_ax.set_title("PS1 residuals")
-
-            cur_ax = ax[0, 1]
-            cur_ax = model.plot(x_max=bin_center[-1], ax=cur_ax)
-            cur_ax.scatter(bin_center, vario)
-            cur_ax.set_xlabel("distance in [m]")
-            cur_ax.set_ylabel("semi-variogram")
-
-            if coord_utm2 is not None:
-                cur_ax = ax[1, 0]
-                sca2 = cur_ax.scatter(x_new, y_new, c=fld_sk_new, vmin=min_val, vmax=max_val)
-                plt.colorbar(sca2, ax=cur_ax, pad=0.03, shrink=0.5)
-                cur_ax.set_title("PS2 prediction of atmospheric effect")
-
-                cur_ax = ax[0, 1]
-                sca4 = cur_ax.scatter(x_new, y_new, c=var_sk_new)
-                plt.colorbar(sca4, ax=cur_ax, pad=0.03, shrink=0.5)
-                cur_ax.set_title("Variance of predicted atmospheric effect")
-
-            plt.close(fig)
+    return idx_range, aps1, model_params
 
-    return idx_range, aps1, aps2
 
-
-def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_utm2: np.ndarray,
+def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndarray,
                                    num_cores: int = 1, bool_plot: bool = False,
                                    logger: Logger) -> tuple[np.ndarray, np.ndarray]:
     """Estimate_atmospheric_phase_screen.
@@ -295,8 +263,6 @@ def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndar
         residual phase (size: num_points1 x num_images)
     coord_utm1: np.ndarray
         coordinates in UTM of the points for which the residuals are given (size: num_points1 x 2)
-    coord_utm2: np.ndarray
-        coordinates in UTM of the new points which shall be interpolated (size: num_points2 x 2)
     num_cores: int
         Number of cores
     bool_plot: bool
@@ -308,8 +274,8 @@ def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndar
     -------
     aps1: np.ndarray
         atmospheric phase screen for the known points (size: num_points1 x num_images)
-    aps2: np.ndarray
-        atmospheric phase screen for the new points (size: num_points2 x num_images)
+    model_params: np.ndarray
+        model parameters for the variogram model (size: num_model_parameters x num_images)
     """
     msg = "#" * 10
     msg += " ESTIMATE ATMOSPHERIC PHASE SCREEN (KRIGING) "
@@ -319,21 +285,20 @@ def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndar
     start_time = time.time()
 
     num_points1 = residuals.shape[0]
-    num_points2 = coord_utm2.shape[0]
     num_time = residuals.shape[1]  # can be either num_ifgs or num_images
 
     bins = gs.variogram.standard_bins(pos=(coord_utm1[:, 1], coord_utm1[:, 0]),
                                       dim=2, latlon=False, mesh_type='unstructured', bin_no=30, max_dist=None)
 
     if num_cores == 1:
-        args = (np.arange(0, num_time), num_time, residuals, coord_utm1, coord_utm2, bins, bool_plot, logger)
-        _, aps1, aps2 = launchSpatialFiltering(parameters=args)
+        args = (np.arange(0, num_time), num_time, residuals, coord_utm1, bins, bool_plot, logger)
+        _, aps1, model_params = launchSpatialFiltering(parameters=args)
     else:
         logger.info(msg="start parallel processing with {} cores.".format(num_cores))
         pool = multiprocessing.Pool(processes=num_cores)
 
         aps1 = np.zeros((num_points1, num_time), dtype=np.float32)
-        aps2 = np.zeros((num_points2, num_time), dtype=np.float32)
+        model_params = np.zeros((4, num_time), dtype=np.float32)   # 4 parameters for the Stable variogram model.
 
         num_cores = num_time if num_cores > num_time else num_cores  # avoids having more samples than cores
         idx = ut.splitDatasetForParallelProcessing(num_samples=num_time, num_cores=num_cores)
@@ -343,7 +308,6 @@ def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndar
             idx_range.shape[0],
             residuals[:, idx_range],
             coord_utm1,
-            coord_utm2,
             bins,
             False,
             logger) for idx_range in idx]
@@ -351,18 +315,17 @@ def estimateAtmosphericPhaseScreen(*, residuals: np.ndarray, coord_utm1: np.ndar
         results = pool.map(func=launchSpatialFiltering, iterable=args)
 
         # retrieve results
-        for i, aps1_i, aps2_i in results:
+        for i, aps1_i, model_params_i in results:
             aps1[:, i] = aps1_i
-            aps2[:, i] = aps2_i
+            model_params[:, i] = model_params_i
 
     m, s = divmod(time.time() - start_time, 60)
     logger.debug(msg='time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
 
-    return aps1, aps2
+    return aps1, model_params
 
 
-def simpleInterpolation(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_utm2: np.ndarray,
-                        interp_method: str = "linear"):
+def simpleInterpolation(*, residuals: np.ndarray, coord_utm1: np.ndarray, interp_method: str = "linear"):
     """SimpleInterpolation.
 
     Simple interpolation of atmospheric phase screen using scipy's griddata function with options "linear" or "cubic".
@@ -374,8 +337,6 @@ def simpleInterpolation(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_
         residual phase (size: num_points x num_ifgs)
     coord_utm1: np.ndarray
         coordinates in UTM of the points for which the residuals are given (size: num_points_p1 x 2)
-    coord_utm2: np.ndarray
-        coordinates in UTM of the new points which shall be interpolated (size: num_points_p2 x 2)
     interp_method: str
         interpolation method (default: "linear"; options: "linear", "cubic")
 
@@ -383,28 +344,14 @@ def simpleInterpolation(*, residuals: np.ndarray, coord_utm1: np.ndarray, coord_
     -------
     aps1: np.ndarray
         atmospheric phase screen for the known points (size: num_points_p1 x num_images)
-    aps2: np.ndarray
-        atmospheric phase screen for the new points (size: num_points_p2 x num_images)
     """
-    num_points2 = coord_utm2.shape[0]
     num_images = residuals.shape[1]
 
     aps1 = np.zeros_like(residuals, dtype=np.float32)
-    aps2 = np.zeros((num_points2, num_images), dtype=np.float32)
     for i in range(num_images):
         aps1[:, i] = griddata(coord_utm1, residuals[:, i], coord_utm1, method=interp_method)
-        aps2[:, i] = griddata(coord_utm1, residuals[:, i], coord_utm2, method=interp_method)
-        # interpolation with 'linear' or 'cubic' yields nan values for pixel that need to be extrapolated.
-        # interpolation with 'knn' solves this problem.
-        mask_extrapolate = np.isnan(aps2[:, i])
-        aps2[mask_extrapolate, i] = griddata(
-            coord_utm1,
-            residuals[:, i],
-            coord_utm2[mask_extrapolate, :],
-            method='nearest'
-        )
 
-    return aps1, aps2
+    return aps1
 
 
 def extractModelParams(model: gs.covmodel.models, logger: Logger):
diff --git a/sarvey/processing.py b/sarvey/processing.py
index dc323f90..acf97085 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -39,11 +39,11 @@
 from mintpy.utils.plot import auto_flip_direction
 
 from sarvey import viewer
-from sarvey.densification import densifyNetwork
+from sarvey.densification import densifyNetwork, selectP2
 from sarvey.filtering import estimateAtmosphericPhaseScreen, simpleInterpolation
 from sarvey.ifg_network import (DelaunayNetwork, SmallBaselineYearlyNetwork, SmallTemporalBaselinesNetwork,
                                 SmallBaselineNetwork, StarNetwork)
-from sarvey.objects import Network, Points, AmplitudeImage, CoordinatesUTM, NetworkParameter, BaseStack
+from sarvey.objects import Network, Points, AmplitudeImage, CoordinatesUTM, NetworkParameter, BaseStack, ApsParameters
 from sarvey.unwrapping import spatialParameterIntegration, \
     parameterBasedNoisyPointRemoval, temporalUnwrapping, spatialUnwrapping, removeGrossOutliers
 from sarvey.preparation import createArcsBetweenPoints, selectPixels, createTimeMaskFromDates
@@ -548,7 +548,6 @@ def runUnwrappingSpace(self):
 
     def runFiltering(self):
         """RunFiltering."""
-        coh_value = int(self.config.filtering.coherence_p2 * 100)
 
         # create output file which contains filtered phase time series
         point1_obj = Points(file_path=join(self.path, "p1_ts_filt.h5"), logger=self.logger)
@@ -634,187 +633,58 @@ def runFiltering(self):
             path_inputs=self.config.data_directories.path_inputs
         )
 
-        # select second-order points
-        cand_mask2 = selectPixels(
-            path=self.path, selection_method="temp_coh",
-            thrsh=self.config.filtering.coherence_p2,
-            grid_size=None, bool_plot=True,
-            logger=self.logger
-        )  # first-order points are included in second-order points
-
-        if self.config.phase_linking.phase_linking:
-            # read PL results
-            pl_coh = readfile.read(join(self.config.phase_linking.path_inverted, "phase_series.h5"),
-                                   datasetName='temporalCoherence')[0]
-            pl_coh = pl_coh[1, :, :]
-            siblings = readfile.read(join(self.config.phase_linking.path_inverted, "phase_series.h5"),
-                                     datasetName='shp')[0]
-
-            if self.config.phase_linking.use_ps:
-                mask_ps = readfile.read(self.config.phase_linking.path_mask_file_ps,
-                                        datasetName='mask')[0].astype(np.bool_)
-                cand_mask_pl = (pl_coh > self.config.filtering.coherence_p2) | mask_ps
-            else:
-                cand_mask_pl = (pl_coh > self.config.filtering.coherence_p2)
-                # remove ps candidates, because the ps detection strategy in miaplpy seems to be biased.
-                cand_mask_pl[siblings <= self.config.phase_linking.num_siblings] = False
-
-            if self.config.phase_linking.spatial_mask_file_pl is not None:
-                path_mask_pl_aoi = join(self.config.phase_linking.spatial_mask_file_pl)
-                self.logger.info(msg="load mask for area of interest from: {}.".format(path_mask_pl_aoi))
-                mask_pl_aoi = readfile.read(path_mask_pl_aoi, datasetName='mask')[0].astype(np.bool_)
-
-                fig = plt.figure(figsize=[15, 5])
-                ax = fig.add_subplot()
-                ax.imshow(mask_pl_aoi, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
-                bmap_obj.plot(ax=ax, logger=self.logger)
-                coord_xy = np.array(np.where(cand_mask_pl)).transpose()
-                val = np.ones_like(cand_mask_pl)
-                sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask_pl], s=0.5,
-                                cmap=plt.get_cmap("autumn_r"),
-                                vmin=1, vmax=2)  # set min, max to ensure that points are yellow
-                cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
-                cbar.ax.set_visible(False)  # make size of axis consistent with all others
-                plt.tight_layout()
-                plt.title("Mask for phase linking points")
-                fig.savefig(join(self.path, "pic", "step_3_mask_coh{}_phase_linking.png".format(coh_value)), dpi=300)
-                plt.close(fig)
-
-                # mask points after plotting, so that available coherent points are visible in figure
-                cand_mask_pl[~mask_pl_aoi] = False
-
-            # combine phase linking coherence with TPC cand_mask2
-            cand_mask2 = cand_mask2 | cand_mask_pl
-
-        mask_valid_area = ut.detectValidAreas(bmap_obj=bmap_obj, logger=self.logger)
-
-        if self.config.filtering.spatial_mask_file_p2 is not None:
-            path_mask_aoi = join(self.config.filtering.spatial_mask_file_p2)
-            self.logger.info(msg="load mask for area of interest from: {}.".format(path_mask_aoi))
-            mask_aoi = readfile.read(path_mask_aoi, datasetName='mask')[0].astype(np.bool_)
-            mask_valid_area &= mask_aoi
-            # todo: add unstable points from p1 for densification
-        else:
-            self.logger.info(msg="No mask for area of interest given.")
-
-        cand_mask2[p1_mask] = True  # add all selected 1.order points to avoid spatial gaps in 2D unwrapping
-        # cand_mask2[cand_mask_sparse] = True  # add only stable points from 1.order points
-
-        cand_mask2 &= mask_valid_area
-
-        fig = plt.figure(figsize=[15, 5])
-        ax = fig.add_subplot()
-        ax.imshow(mask_valid_area, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
-        bmap_obj.plot(ax=ax, logger=self.logger)
-        coord_xy = np.array(np.where(cand_mask2)).transpose()
-        val = np.ones_like(cand_mask2)
-        sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask2], s=0.5, cmap=plt.get_cmap("autumn_r"),
-                        vmin=1, vmax=2)  # set min, max to ensure that points are yellow
-        cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
-        cbar.ax.set_visible(False)  # make size of axis consistent with all others
-        plt.tight_layout()
-        plt.title("Mask for dense point set")
-        fig.savefig(join(self.path, "pic", "step_3_mask_coh{}.png".format(coh_value)), dpi=300)
-        plt.close(fig)
-
-        point2_obj = Points(file_path=join(self.path, "coh{}_ifg_wr.h5".format(coh_value)), logger=self.logger)
-        coord_xy = np.array(np.where(cand_mask2)).transpose()
-        point_id2 = point_id_img[cand_mask2]
-        point2_obj.prepare(
-            point_id=point_id2,
-            coord_xy=coord_xy,
-            path_inputs=self.config.data_directories.path_inputs
-        )
-
-        ifg_stack_obj = BaseStack(file=join(self.path, "ifg_stack.h5"), logger=self.logger)
-
-        point2_obj.phase = ut.readPhasePatchwise(stack_obj=ifg_stack_obj, dataset_name="ifgs",
-                                                 num_patches=self.config.processing.num_patches, cand_mask=cand_mask2,
-                                                 point_id_img=point_id_img, logger=self.logger)
-
-        if self.config.phase_linking.phase_linking:
-            self.logger.info(msg="read phase from MiaplPy results...")
-            phase_linking_obj = BaseStack(file=join(self.config.phase_linking.path_inverted, "phase_series.h5"),
-                                          logger=self.logger)
-
-            pl_phase = ut.readPhasePatchwise(
-                stack_obj=phase_linking_obj, dataset_name="phase",
-                num_patches=self.config.processing.num_patches,
-                cand_mask=cand_mask2,
-                point_id_img=point_id_img, logger=self.logger
-            )
-
-            # subset to time span
-            slc_stack_obj = slcStack(join(self.config.data_directories.path_inputs, "slcStack.h5"))
-            slc_stack_obj.open()
-            time_mask = createTimeMaskFromDates(
-                start_date=self.config.preparation.start_date,
-                stop_date=self.config.preparation.stop_date,
-                date_list=slc_stack_obj.dateList,
-                logger=self.logger
-            )[0]
-            pl_phase = pl_phase[:, time_mask]
-
-            pl_ifgs = np.zeros((point2_obj.num_points, point2_obj.ifg_net_obj.num_ifgs), dtype=np.float32)
-
-            c = 0
-            for i, j in np.asarray(point1_obj.ifg_net_obj.ifg_list):
-                pl_ifgs[:, c] = np.angle(np.exp(1j * pl_phase[:, i]) * np.conjugate(np.exp(1j * pl_phase[:, j])))
-                c += 1
-
-            # change only phase for good phase linking pixels and keep original phase for good tpc pixels
-            mask_pl = cand_mask_pl[cand_mask2]
-            point2_obj.phase[mask_pl] = pl_ifgs[mask_pl]
-
-        point2_obj.writeToFile()
-        del point2_obj, ifg_stack_obj
-
-        aps2_obj = Points(file_path=join(self.path, "coh{}_aps.h5".format(coh_value)), logger=self.logger)
-        aps2_obj.open(
-            other_file_path=join(self.path, "coh{}_ifg_wr.h5".format(coh_value)),
-            path_inputs=self.config.data_directories.path_inputs
-        )
+        # select second-order points moved to selectP2()
 
         if self.config.filtering.skip_filtering:
             msg = "#" * 10
             msg += " SKIP ATMOSPHERIC FILTERING! "
             msg += "#" * 10
             self.logger.info(msg=msg)
+            model_name = "None"
+            aps_model_params = None
             num_points1 = phase_for_aps_filtering.shape[0]
-            num_points2 = aps2_obj.coord_utm.shape[0]
             num_time = phase_for_aps_filtering.shape[1]
             aps1_phase = np.zeros((num_points1, num_time), dtype=np.float32)
-            aps2_phase = np.zeros((num_points2, num_time), dtype=np.float32)
         else:
             # spatial filtering of points with linear motion only (no non-linear motion)
             if self.config.filtering.interpolation_method == "kriging":
-                aps1_phase, aps2_phase = estimateAtmosphericPhaseScreen(
+                model_name = "Stable"  # To be integrated in the config?
+                aps1_phase, aps_model_params = estimateAtmosphericPhaseScreen(
                     residuals=phase_for_aps_filtering,
                     coord_utm1=point1_obj.coord_utm,
-                    coord_utm2=aps2_obj.coord_utm,
                     num_cores=self.config.processing.num_cores,
                     bool_plot=False,
                     logger=self.logger
                 )
             else:
-                aps1_phase, aps2_phase = simpleInterpolation(
+                model_name = "Linear"  # To be integrated in the config?
+                aps_model_params = None
+                aps1_phase = simpleInterpolation(
                     residuals=phase_for_aps_filtering,
                     coord_utm1=point1_obj.coord_utm,
-                    coord_utm2=aps2_obj.coord_utm,
                     interp_method=self.config.filtering.interpolation_method
                 )
 
+        # save APS parameters for later use in step 4
+        aps_file = join(self.path, "aps_parameters.h5")
+        self.logger.debug(f"Prepare Aps Parameters file {aps_file}")
+        aps_params_obj = ApsParameters(file_path=aps_file, logger=self.logger)
+        aps_params_obj.prepare(model_name=model_name, model_params=aps_model_params,
+                               phase=phase_for_aps_filtering)
+
         point1_obj.phase -= aps1_phase
         point1_obj.writeToFile()
 
         aps1_obj.phase = aps1_phase
-        aps2_obj.phase = aps2_phase
         aps1_obj.writeToFile()
-        aps2_obj.writeToFile()
 
     def runDensificationTimeAndSpace(self):
         """RunDensificationTimeAndSpace."""
+        coherence_p2 = self.config.filtering.coherence_p2
         coh_value = int(self.config.filtering.coherence_p2 * 100)
+        self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")
+
+        _, aps2_obj = selectP2(output_path=self.path, config=self.config, logger=self.logger)
 
         point2_obj = Points(file_path=join(self.path, "coh{}_ifg_unw.h5".format(coh_value)), logger=self.logger)
         point2_obj.open(
@@ -834,9 +704,6 @@ def runDensificationTimeAndSpace(self):
         aps1_obj = Points(file_path=join(self.path, "p1_aps.h5"), logger=self.logger)
         aps1_obj.open(path_inputs=self.config.data_directories.path_inputs)
 
-        aps2_obj = Points(file_path=join(self.path, "coh{}_aps.h5".format(coh_value)), logger=self.logger)
-        aps2_obj.open(path_inputs=self.config.data_directories.path_inputs)
-
         if self.config.filtering.spatial_mask_file_p2 is None:
             """
             overview of points contained in the *_obj
@@ -1090,7 +957,11 @@ def runDensificationTimeAndSpace(self):
 
     def runDensificationSpace(self):
         """RunDensification."""
-        coh_value = int(self.config.filtering.coherence_p2 * 100)
+        coherence_p2 = self.config.filtering.coherence_p2
+        coh_value = int(coherence_p2 * 100)
+
+        self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")
+        _, aps2_obj = selectP2(output_path=self.path, config=self.config, logger=self.logger)
 
         point_obj = Points(file_path=join(self.path, "coh{}_ifg_unw.h5".format(coh_value)), logger=self.logger)
         point_obj.open(
@@ -1098,9 +969,6 @@ def runDensificationSpace(self):
             path_inputs=self.config.data_directories.path_inputs
         )  # open wr phase
 
-        aps2_obj = Points(file_path=join(self.path, "coh{}_aps.h5".format(coh_value)), logger=self.logger)
-        aps2_obj.open(path_inputs=self.config.data_directories.path_inputs)
-
         # return to ifg-space
         a_ifg = point_obj.ifg_net_obj.getDesignMatrix()
         aps2_ifg_phase = np.matmul(a_ifg, aps2_obj.phase.T).T

From 0c6e18297ea327dea6e0c0fc9ed941ca09edd559 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 14:28:58 +0200
Subject: [PATCH 13/61] fix lint

---
 sarvey/densification.py | 2 +-
 sarvey/filtering.py     | 3 ++-
 sarvey/processing.py    | 1 -
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 0c94685f..06807f8f 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -352,7 +352,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     # this function is mainly adapted from sarvey.processing.runFiltering
     # TODO: Directly pass input parameters instead of config dictionary
     coherence_p2 = config.filtering.coherence_p2
-    logger.debug(f"Start Selecting 2nd order points.")
+    logger.debug("Start Selecting 2nd order points.")
 
     coh_value = int(coherence_p2 * 100)
 
diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index bbdb90b8..dd9ed25d 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -204,7 +204,8 @@ def launchSpatialFiltering(parameters: tuple):
         model = gs.Stable(dim=2)
         params = extractModelParams(model, logger)
         model_params = np.array([np.full(num_time, param) for param in params], dtype=np.float32)
-    else:  #  TODO: Other models to be added later
+    else:
+        # TODO: Other models to be added later
         error_msg = f"Model {model_name} not implemented yet."
         logger.error(msg=error_msg)
         raise ValueError(error_msg)
diff --git a/sarvey/processing.py b/sarvey/processing.py
index acf97085..1edc9bfa 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -555,7 +555,6 @@ def runFiltering(self):
             other_file_path=join(self.path, "p1_ts.h5"),
             path_inputs=self.config.data_directories.path_inputs
         )
-        p1_mask = point1_obj.createMask()  # used later for selecting psCand2 when a spatial mask AOI is given.
 
         # select only pixels which have low phase noise and are well distributed
         mask = point1_obj.createMask()

From 3ce79444dafd983013162a906e9ef49fb7bc2e59 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 14:31:45 +0200
Subject: [PATCH 14/61] fix lint

---
 sarvey/densification.py | 5 ++---
 sarvey/filtering.py     | 1 -
 sarvey/processing.py    | 1 -
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 06807f8f..d90327d1 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -330,7 +330,7 @@ def densifyNetwork(*, point1_obj: Points, vel_p1: np.ndarray, demerr_p1: np.ndar
 
 
 def selectP2(*, output_path: str, config: dict, logger: Logger):
-    """Select second order points and interpolate APS to them
+    """Select second order points and interpolate APS to them.
 
     Parameters
     ----------
@@ -341,13 +341,12 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     logger : Logger
         Logger instance for logging messages.
 
-    Returns:
+    Returns
     -------
     point2_obj : Points
         Points object with second-order points
     aps2_obj: Points
         Points object with APS estimation for second-order points
-
     """
     # this function is mainly adapted from sarvey.processing.runFiltering
     # TODO: Directly pass input parameters instead of config dictionary
diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index dd9ed25d..de82f89b 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -411,7 +411,6 @@ def applyModelParams(model: gs.covmodel.models, params: tuple, logger: Logger):
     ValueError
         If the model is not implemented.
     """
-
     if model.name == 'Stable':
         model.var, model.len_scale, model.nugget, model.alpha = params
     # elif model.name == 'Gaussian':  # TODO: implement other models
diff --git a/sarvey/processing.py b/sarvey/processing.py
index 1edc9bfa..de747f92 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -548,7 +548,6 @@ def runUnwrappingSpace(self):
 
     def runFiltering(self):
         """RunFiltering."""
-
         # create output file which contains filtered phase time series
         point1_obj = Points(file_path=join(self.path, "p1_ts_filt.h5"), logger=self.logger)
         point1_obj.open(

From 1af045c660c89a969364e7d54202384d3aee67fd Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 15:38:08 +0200
Subject: [PATCH 15/61] update docs

---
 docs/processing.rst | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/docs/processing.rst b/docs/processing.rst
index a2d8a2c7..78caa804 100644
--- a/docs/processing.rst
+++ b/docs/processing.rst
@@ -221,9 +221,7 @@ Step 3: Filtering
 ^^^^^^^^^^^^^^^^^
 
 In this step, the atmospheric phase screen (APS) is estimated from the displacement time series of the first-order points.
-Afterwards, the APS is interpolated to the location of the second-order points.
 The filtering can be skipped by setting **filtering:skip_filtering** to True.
-However, the step 3 has to be executed as the second-order points are selected during this step.
 
 - Selecting pixels with no or linear displacement:
     Among the first-order points, the points with no or merely linear displacement are selected (**filtering:use_moving_points**).
@@ -231,44 +229,47 @@ However, the step 3 has to be executed as the second-order points are selected d
     Points with a non-linear displacement behaviour are removed by a threshold on the temporal autocorrelation of the displacement time series (**filtering:max_auto_corr**) (Crosetto et al. 2018).
     A regular grid (**filtering:grid_size** in [m]) is applied to select the first-order points with the lowest temporal autocorrelation to reduce the computational complexity during filtering.
 
-- Selecting second-order points:
-    Second-order points are selected based on a temporal coherence threshold (**filtering:coherence_p2**) on the temporal phase coherence computed during step 0.
-    A mask file can be specified (**filtering:spatial_mask_file_p2**) to limit the second-order points to the given area of interest.
-    Second-order points can also be selected based on the results of phase-linking (set **phase_linking:phase_linking** to True) implemented in MiaplPy (Mirzaee et al. 2023).
-    More information on Miaplpy and phase-linking can be found `here <preparation>`_.
-    The number of siblings (**phase_linking:num_siblings**) used during phase-linking within MiaplPy processing needs to be specified to identify the distributed scatterers (DS) among the pixels selected by MiaplPy.
-    A mask file can be specified (**phase_linking:spatial_mask_file_pl**) to limit the phase-linking to the given area of interest.
-    MiaplPy also provides a selection of persistent scatterers (PS) which can be included as second-order points (set **phase_linking:use_ps** to True).
-    In case the second-order points are selected among the results from MiaplPy, the filtered interferometric phase (MiaplPy result) is used for the respective points.
-    The DS pixels from MiaplPy and the pixels selected with the temporal phase coherence from step 0 are both selected with the same coherence threshold (**filtering:coherence_p2**).
-
 - Estimating the atmospheric phase screen (APS):
     The estimation of the APS takes place in time-domain and not interferogram-domain to reduce the computational time.
     The phase contributions are removed from the first-order points which were selected for atmospheric filtering.
     Their residual time series contains atmospheric phase contributions and noise.
     As the APS is assumed to be spatially correlated, the residuals of all points are spatially filtered (**filtering:interpolation_method**) independently for each time step.
-    After filtering, the estimated APS is interpolated to the location of the second-order points.
 
 - Output of this step
     - p1_ts_filt.h5
     - p1_aps.h5
-    - cohXX_aps.h5
-    - cohXX_ifg_wr.h5
+    - aps_parameters.h5
 
-The placeholder XX depends on the threshold for the temporal coherence used for selecting the second-order points.
-For example, a threshold of 0.8 would result in coh80_aps.h5 and coh80_ifg_wr.h5.
 
 Step 4: Densification
 ^^^^^^^^^^^^^^^^^^^^^
 
+In this step, second order points are selected. Afterwards, the APS, estimated in step 3, is interpolated to the location of the second-order points.
+
 Two unwrapping options (**processing:temporal_unwrapping**, also applies to step 2) are implemented and should be chosen based on the characteristics of the displacement (spatial extend, magnitude, temporal behaviour).
 
+- Selecting second-order points:
+    Second-order points are selected based on a temporal coherence threshold (**filtering:coherence_p2**) on the temporal phase coherence computed during step 0.
+    A mask file can be specified (**filtering:spatial_mask_file_p2**) to limit the second-order points to the given area of interest.
+    Second-order points can also be selected based on the results of phase-linking (set **phase_linking:phase_linking** to True) implemented in MiaplPy (Mirzaee et al. 2023).
+    More information on Miaplpy and phase-linking can be found `here <preparation>`_.
+    The number of siblings (**phase_linking:num_siblings**) used during phase-linking within MiaplPy processing needs to be specified to identify the distributed scatterers (DS) among the pixels selected by MiaplPy.
+    A mask file can be specified (**phase_linking:spatial_mask_file_pl**) to limit the phase-linking to the given area of interest.
+    MiaplPy also provides a selection of persistent scatterers (PS) which can be included as second-order points (set **phase_linking:use_ps** to True).
+    In case the second-order points are selected among the results from MiaplPy, the filtered interferometric phase (MiaplPy result) is used for the respective points.
+    The DS pixels from MiaplPy and the pixels selected with the temporal phase coherence from step 0 are both selected with the same coherence threshold (**filtering:coherence_p2**).
+
+- Apply estimated atmospheric phase screen (APS) to second-order points:
+    The APS, estimated in step 3, is interpolated to the location of the second-order points.
+
 - Output of this step
+    - cohXX_ifg_wr.h5
+    - cohXX_aps.h5
     - cohXX_ifg_unw.h5
     - cohXX_ts.h5
 
-The placeholder XX depends on the threshold for the temporal coherence used for selecting the second-order points during filtering in step 3.
-For example, a threshold of 0.8 would result in coh80_ifg_unw.h5 and coh80_ts.h5.
+The placeholder XX depends on the threshold for the temporal coherence used for selecting the second-order points.
+For example, a threshold of 0.8 would result in coh80_aps.h5 and coh80_ifg_wr.h5.
 
 Option 1: Unwrapping in time and space
 """"""""""""""""""""""""""""""""""""""

From 81b5a6c0daaab5dadf0bdab1f93718b8eb5607ef Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 15:38:16 +0200
Subject: [PATCH 16/61] update tests

---
 tests/test_processing.py | 44 ++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/tests/test_processing.py b/tests/test_processing.py
index 704c4c7b..2482deb6 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -150,11 +150,8 @@ def testUnwrappingTimeAndSpace(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        coh_value = int(config.filtering.coherence_p2 * 100)
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
-            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
-            f'Processing failed (coh{coh_value}_aps.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
+            f'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -162,6 +159,10 @@ def testUnwrappingTimeAndSpace(self):
 
         run(config=config, args=args, logger=self.logger)
         coh_value = int(config.filtering.coherence_p2 * 100)
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
+            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
+            f'Processing failed (coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ts.h5")), \
@@ -223,11 +224,8 @@ def testUnwrappingSpace(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        coh_value = int(config.filtering.coherence_p2 * 100)
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
-            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
-            f'Processing failed (coh{coh_value}_aps.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
+            f'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -235,6 +233,10 @@ def testUnwrappingSpace(self):
 
         run(config=config, args=args, logger=self.logger)
         coh_value = int(config.filtering.coherence_p2 * 100)
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
+            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
+            f'Processing failed (coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ts.h5")), \
@@ -298,11 +300,8 @@ def testPhaseLinking(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        coh_value = int(config.filtering.coherence_p2 * 100)
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
-            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
-            f'Processing failed (coh{coh_value}_aps.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
+            f'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -310,6 +309,10 @@ def testPhaseLinking(self):
 
         run(config=config, args=args, logger=self.logger)
         coh_value = int(config.filtering.coherence_p2 * 100)
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
+            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
+            f'Processing failed (coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ts.h5")), \
@@ -373,11 +376,8 @@ def testMasking(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        coh_value = int(config.filtering.coherence_p2 * 100)
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
-            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
-            f'Processing failed (coh{coh_value}_aps.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
+            f'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -385,6 +385,10 @@ def testMasking(self):
 
         run(config=config, args=args, logger=self.logger)
         coh_value = int(config.filtering.coherence_p2 * 100)
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
+            f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
+            f'Processing failed (coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ts.h5")), \

From aefff8229820b8da0f0b7ff114bb89d65eed0c41 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 17:43:52 +0200
Subject: [PATCH 17/61] move filtering.coherence_p2 to
 densification.coherence_p2

---
 sarvey/config.py        | 30 +++++++++++++++---------------
 sarvey/densification.py |  2 +-
 sarvey/processing.py    |  6 +++---
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/sarvey/config.py b/sarvey/config.py
index c56d5106..bd5ea43c 100644
--- a/sarvey/config.py
+++ b/sarvey/config.py
@@ -474,12 +474,6 @@ class Filtering(BaseModel, extra=Extra.forbid):
         default="kriging"
     )
 
-    coherence_p2: float = Field(
-        title="Temporal coherence threshold",
-        description="Set the temporal coherence threshold for the filtering step.",
-        default=0.8
-    )
-
     grid_size: int = Field(
         title="Grid size [m].",
         description="Set the grid size for spatial filtering.",
@@ -504,15 +498,6 @@ class Filtering(BaseModel, extra=Extra.forbid):
         default=0.3
     )
 
-    @validator('coherence_p2')
-    def checkTempCohThrsh2(cls, v):
-        """Check if the temporal coherence threshold is valid."""
-        if v < 0:
-            raise ValueError("Temporal coherence threshold cannot be negative.")
-        if v > 1:
-            raise ValueError("Temporal coherence threshold cannot be greater than 1.")
-        return v
-
     @validator('interpolation_method')
     def checkInterpolationMethod(cls, v):
         """Check if the interpolation method is valid."""
@@ -550,6 +535,12 @@ def checkMaxAutoCorr(cls, v):
 class Densification(BaseModel, extra=Extra.forbid):
     """Template for densification settings in config file."""
 
+    coherence_p2: float = Field(
+        title="Temporal coherence threshold",
+        description="Set the temporal coherence threshold for the densification step.",
+        default=0.8
+    )
+
     coherence_threshold: float = Field(
         title="Coherence threshold for densification",
         description="Set coherence threshold for densification.",
@@ -601,6 +592,15 @@ class Densification(BaseModel, extra=Extra.forbid):
         default=1
     )
 
+    @validator('coherence_p2')
+    def checkTempCohThrsh2(cls, v):
+        """Check if the temporal coherence threshold is valid."""
+        if v < 0:
+            raise ValueError("Temporal coherence threshold cannot be negative.")
+        if v > 1:
+            raise ValueError("Temporal coherence threshold cannot be greater than 1.")
+        return v
+
     @validator('coherence_threshold')
     def checkCoherenceThresh(cls, v):
         """Check if coherence_threshold is valid."""
diff --git a/sarvey/densification.py b/sarvey/densification.py
index d90327d1..034307b0 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -350,7 +350,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     """
     # this function is mainly adapted from sarvey.processing.runFiltering
     # TODO: Directly pass input parameters instead of config dictionary
-    coherence_p2 = config.filtering.coherence_p2
+    coherence_p2 = config.densification.coherence_p2
     logger.debug("Start Selecting 2nd order points.")
 
     coh_value = int(coherence_p2 * 100)
diff --git a/sarvey/processing.py b/sarvey/processing.py
index de747f92..e3d2061e 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -678,8 +678,8 @@ def runFiltering(self):
 
     def runDensificationTimeAndSpace(self):
         """RunDensificationTimeAndSpace."""
-        coherence_p2 = self.config.filtering.coherence_p2
-        coh_value = int(self.config.filtering.coherence_p2 * 100)
+        coherence_p2 = self.config.densification.coherence_p2
+        coh_value = int(self.config.densification.coherence_p2 * 100)
         self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")
 
         _, aps2_obj = selectP2(output_path=self.path, config=self.config, logger=self.logger)
@@ -955,7 +955,7 @@ def runDensificationTimeAndSpace(self):
 
     def runDensificationSpace(self):
         """RunDensification."""
-        coherence_p2 = self.config.filtering.coherence_p2
+        coherence_p2 = self.config.densification.coherence_p2
         coh_value = int(coherence_p2 * 100)
 
         self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")

From 2ee3857381ebb72b64d93a866a4d72c813421ee3 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 17:44:25 +0200
Subject: [PATCH 18/61] update test

---
 tests/test_processing.py        | 10 +++++-----
 tests/testdata/config_test.json |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_processing.py b/tests/test_processing.py
index 2482deb6..1d296a3f 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -158,7 +158,7 @@ def testUnwrappingTimeAndSpace(self):
         args.stop = 4
 
         run(config=config, args=args, logger=self.logger)
-        coh_value = int(config.filtering.coherence_p2 * 100)
+        coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
@@ -232,7 +232,7 @@ def testUnwrappingSpace(self):
         args.stop = 4
 
         run(config=config, args=args, logger=self.logger)
-        coh_value = int(config.filtering.coherence_p2 * 100)
+        coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
@@ -251,7 +251,7 @@ def testPhaseLinking(self):
         config.phase_linking.phase_linking = True
         config.phase_linking.use_ps = True
         config.processing.temporal_unwrapping = True
-        config.filtering.coherence_p2 = 0.75
+        config.densification.coherence_p2 = 0.75
 
         # preparation
         args.start = 0
@@ -308,7 +308,7 @@ def testPhaseLinking(self):
         args.stop = 4
 
         run(config=config, args=args, logger=self.logger)
-        coh_value = int(config.filtering.coherence_p2 * 100)
+        coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
@@ -384,7 +384,7 @@ def testMasking(self):
         args.stop = 4
 
         run(config=config, args=args, logger=self.logger)
-        coh_value = int(config.filtering.coherence_p2 * 100)
+        coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (coh{coh_value}_ifg_wr.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, f"coh{coh_value}_aps.h5")), \
diff --git a/tests/testdata/config_test.json b/tests/testdata/config_test.json
index 5ec4e598..14c8e4ec 100755
--- a/tests/testdata/config_test.json
+++ b/tests/testdata/config_test.json
@@ -44,13 +44,13 @@
     "filtering": {
         "skip_filtering": false,
         "interpolation_method": "kriging",
-        "coherence_p2": 0.9,
         "grid_size": 1000,
         "spatial_mask_file_p2": null,
         "use_moving_points": true,
         "max_auto_corr": 0.3
     },
     "densification": {
+        "coherence_p2": 0.9,
         "coherence_threshold": 0.5,
         "num_connections_p1": 5,
         "num_connections_p2": 10,

From 6a9d23706337b3f7b517b1a74366a2840c5a0186 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 17:52:58 +0200
Subject: [PATCH 19/61] update savefig filename step_3 to step_4

---
 sarvey/densification.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 034307b0..a742b72b 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -416,7 +416,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
             cbar.ax.set_visible(False)  # make size of axis consistent with all others
             plt.tight_layout()
             plt.title("Mask for phase linking points")
-            fig.savefig(join(output_path, "pic", "step_3_mask_coh{}_phase_linking.png".format(coh_value)), dpi=300)
+            fig.savefig(join(output_path, "pic", "step_4_mask_coh{}_phase_linking.png".format(coh_value)), dpi=300)
             plt.close(fig)
 
             # mask points after plotting, so that available coherent points are visible in figure
@@ -456,7 +456,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     cbar.ax.set_visible(False)  # make size of axis consistent with all others
     plt.tight_layout()
     plt.title("Mask for dense point set")
-    fig.savefig(join(output_path, "pic", "step_3_mask_coh{}.png".format(coh_value)), dpi=300)
+    fig.savefig(join(output_path, "pic", "step_4_mask_coh{}.png".format(coh_value)), dpi=300)
     plt.close(fig)
 
     p2_file = join(output_path, f"coh{coh_value}_ifg_wr.h5")

From b84d47f917dfe2c1bd88f4d7c0c4945507dc7c6a Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 18:05:02 +0200
Subject: [PATCH 20/61] update docs processing.rst

---
 docs/processing.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/processing.rst b/docs/processing.rst
index 78caa804..36d5205a 100644
--- a/docs/processing.rst
+++ b/docs/processing.rst
@@ -118,7 +118,7 @@ Step 0: Preparation
 - Estimating the temporal coherence:
     The phase noise of each pixel is approximated by the estimation of the temporal phase coherence (Zhao and Mallorqui 2019).
     Thereby, a low-pass filter with a certain window size is used (**preparation:filter_wdw_size**).
-    The temporal coherence is used to select the first- and second-order points in the later steps (**consistency_check:coherence_p1** and **filtering:coherence_p2**).
+    The temporal coherence is used to select the first- and second-order points in the later steps (**consistency_check:coherence_p1** and **densification:coherence_p2**).
 
 - Output of this step
     - background_map.h5
@@ -249,7 +249,7 @@ In this step, second order points are selected. Afterwards, the APS, estimated i
 Two unwrapping options (**processing:temporal_unwrapping**, also applies to step 2) are implemented and should be chosen based on the characteristics of the displacement (spatial extend, magnitude, temporal behaviour).
 
 - Selecting second-order points:
-    Second-order points are selected based on a temporal coherence threshold (**filtering:coherence_p2**) on the temporal phase coherence computed during step 0.
+    Second-order points are selected based on a temporal coherence threshold (**densification:coherence_p2**) on the temporal phase coherence computed during step 0.
     A mask file can be specified (**filtering:spatial_mask_file_p2**) to limit the second-order points to the given area of interest.
     Second-order points can also be selected based on the results of phase-linking (set **phase_linking:phase_linking** to True) implemented in MiaplPy (Mirzaee et al. 2023).
     More information on Miaplpy and phase-linking can be found `here <preparation>`_.
@@ -257,7 +257,7 @@ Two unwrapping options (**processing:temporal_unwrapping**, also applies to step
     A mask file can be specified (**phase_linking:spatial_mask_file_pl**) to limit the phase-linking to the given area of interest.
     MiaplPy also provides a selection of persistent scatterers (PS) which can be included as second-order points (set **phase_linking:use_ps** to True).
     In case the second-order points are selected among the results from MiaplPy, the filtered interferometric phase (MiaplPy result) is used for the respective points.
-    The DS pixels from MiaplPy and the pixels selected with the temporal phase coherence from step 0 are both selected with the same coherence threshold (**filtering:coherence_p2**).
+    The DS pixels from MiaplPy and the pixels selected with the temporal phase coherence from step 0 are both selected with the same coherence threshold (**densification:coherence_p2**).
 
 - Apply estimated atmospheric phase screen (APS) to second-order points:
     The APS, estimated in step 3, is interpolated to the location of the second-order points.

From 466e64d610947281404573b3764dfbeeb92ed8d3 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sun, 4 Aug 2024 18:05:30 +0200
Subject: [PATCH 21/61] update docs demo

---
 docs/demo/demo_masjed_dam_detailed_guide.rst | 24 +++++++++++++-------
 docs/demo/demo_masjed_dam_fast_track.rst     |  2 +-
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/docs/demo/demo_masjed_dam_detailed_guide.rst b/docs/demo/demo_masjed_dam_detailed_guide.rst
index 14b60ada..205b52a5 100644
--- a/docs/demo/demo_masjed_dam_detailed_guide.rst
+++ b/docs/demo/demo_masjed_dam_detailed_guide.rst
@@ -261,15 +261,12 @@ Outputs of this step are:
 .. code-block:: none
 
     outputs/
-    ├── coh80_ifg_wr.h5
-    ├── coh80_aps.h5
     ├── p1_aps.h5
     ├── p1_ts_filt.h5
+    ├── aps_parameters.h5
     └── pic/
         ├── step_3_temporal_autocorrelation.png
-        ├── step_3_stable_points.png
-        ├── selected_pixels_temp_coh_0.8.png
-        └── step_3_mask_coh80.png
+        └── step_3_stable_points.png
 
 
 Step 2.4: Run Step 4 of SARvey
@@ -279,7 +276,18 @@ Step 2.4: Run Step 4 of SARvey
 
     sarvey -f config.json 4 4
 
-.. outputs directory structure to be added
+Outputs of this step are:
+
+.. code-block:: none
+
+    outputs/
+    ├── coh80_ifg_wr.h5
+    ├── coh80_aps.h5
+    ├── coh80_ifg_unw.h5
+    ├── coh80_ts.h5
+    └── pic/
+        ├── selected_pixels_temp_coh_0.8.png
+        └── step_4_mask_coh80.png
 
 
 The results of step 4 of SARvey, including the time series, are stored in the `coh80_ts.h5` file. The file is named based on the `coherence_p2` parameter in the config.json file.
@@ -310,11 +318,11 @@ Step 4: Modify Config File and Rerun SARvey
 
 Modify the config.json file and change **coherence_p2** from 0.8 to 0.7.
 
-Run steps 3 and 4 using the following command:
+Rerun steps 4 using the following command:
 
 .. code-block:: bash
 
-    sarvey -f config.json 3 4
+    sarvey -f config.json 4 4
 
 
 A new file `coh70_ts.h5` is created. You can now visualize this file that has a higher point density.
diff --git a/docs/demo/demo_masjed_dam_fast_track.rst b/docs/demo/demo_masjed_dam_fast_track.rst
index 372a0794..d5408a52 100644
--- a/docs/demo/demo_masjed_dam_fast_track.rst
+++ b/docs/demo/demo_masjed_dam_fast_track.rst
@@ -51,7 +51,7 @@ You can run each step individually or a range of steps by specifying the first a
 Check Outputs
 """""""""""""
 
-First, check the output snapshots in the `outputs/pics` directory. You can also use **`sarvey_plot`** to plot various products to assess the quality of the results and decide how to adjust parameters.  Modify the parameters in the config file and rerun the corresponding steps of `sarvey` to improve the results. For instance, changing **`coherence_p2`** from 0.8 to 0.7 and rerunning steps 3 and 4 can increase the density of the final set of points. However, be cautious that reducing the value too much may include noisy points of low quality in the analysis, potentially leading to poor final results. You can check the details of all parameters using the -p flag in `sarvey` and decide how to tune them. For more explanations, please refer to :ref:`processing`
+First, check the output snapshots in the `outputs/pics` directory. You can also use **`sarvey_plot`** to plot various products to assess the quality of the results and decide how to adjust parameters.  Modify the parameters in the config file and rerun the corresponding steps of `sarvey` to improve the results. For instance, changing **`coherence_p2`** from 0.8 to 0.7 and rerunning step 4 can increase the density of the final set of points. However, be cautious that reducing the value too much may include noisy points of low quality in the analysis, potentially leading to poor final results. You can check the details of all parameters using the -p flag in `sarvey` and decide how to tune them. For more explanations, please refer to :ref:`processing`
 
 
 

From d8f6505121d0e621ec00f0c247455d1ce4477915 Mon Sep 17 00:00:00 2001
From: piter <piter@ipi.uni-hannover.de>
Date: Tue, 6 Aug 2024 16:07:27 +0200
Subject: [PATCH 22/61] Change type hint from dict to Config.

---
 sarvey/densification.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index a742b72b..28063181 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -44,6 +44,7 @@
 import sarvey.utils as ut
 from sarvey.preparation import selectPixels, createTimeMaskFromDates
 from sarvey.filtering import applySimpleInterpolationToP2, applySpatialFilteringToP2
+from sarvey.config import Config
 
 
 def densificationInitializer(tree_p1: KDTree, tree_p2: KDTree, point2_obj: Points, demod_phase1: np.ndarray):
@@ -329,7 +330,7 @@ def densifyNetwork(*, point1_obj: Points, vel_p1: np.ndarray, demerr_p1: np.ndar
     return demerr_p2, vel_p2, gamma_p2, mean_gamma
 
 
-def selectP2(*, output_path: str, config: dict, logger: Logger):
+def selectP2(*, output_path: str, config: Config, logger: Logger):
     """Select second order points and interpolate APS to them.
 
     Parameters
@@ -403,7 +404,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
             logger.debug(f"Load mask for area of interest from file {path_mask_pl_aoi}.")
             mask_pl_aoi = readfile.read(path_mask_pl_aoi, datasetName='mask')[0].astype(np.bool_)
 
-            fig = plt.figure(figsize=[15, 5])
+            fig = plt.figure(figsize=(15, 5))
             ax = fig.add_subplot()
             ax.imshow(mask_pl_aoi, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
             bmap_obj.plot(ax=ax, logger=logger)
@@ -444,7 +445,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
     cand_mask2 &= mask_valid_area
     logger.info(f"Final number of selected second-order points: {cand_mask2.sum()}.")
 
-    fig = plt.figure(figsize=[15, 5])
+    fig = plt.figure(figsize=(15, 5))
     ax = fig.add_subplot()
     ax.imshow(mask_valid_area, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
     bmap_obj.plot(ax=ax, logger=logger)
@@ -528,9 +529,7 @@ def selectP2(*, output_path: str, config: dict, logger: Logger):
 
     if config.filtering.skip_filtering:
         logger.info("Skip APS filtering")
-        # TODO: check if this is correct ####
-        # original code: aps2_phase = np.zeros_like(point2_obj.phase)
-        aps2_phase = np.zeros((point2_obj.num_points, aps_params_obj.phase.shape[1]))
+        aps2_phase = np.zeros((point2_obj.num_points, point2_obj.ifg_net_obj.num_images))
     else:
         logger.debug("Apply APS filtering to second-order points.")
         if config.filtering.interpolation_method == "kriging":

From 27253dc7dd0b929bb5cc1fb91d279e9a96853a11 Mon Sep 17 00:00:00 2001
From: piter <piter@ipi.uni-hannover.de>
Date: Tue, 6 Aug 2024 16:09:24 +0200
Subject: [PATCH 23/61] Fix lint.

---
 tests/test_processing.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/test_processing.py b/tests/test_processing.py
index 1d296a3f..ebcbf84c 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -150,8 +150,8 @@ def testUnwrappingTimeAndSpace(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
-            f'Processing failed (aps_parameters.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, "aps_parameters.h5")), \
+            'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -224,8 +224,8 @@ def testUnwrappingSpace(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
-            f'Processing failed (aps_parameters.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, "aps_parameters.h5")), \
+            'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -300,8 +300,8 @@ def testPhaseLinking(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
-            f'Processing failed (aps_parameters.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, "aps_parameters.h5")), \
+            'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4
@@ -376,8 +376,8 @@ def testMasking(self):
             'Processing failed (p1_ts_filt.h5 not created).'
         assert glob(os.path.join(config.data_directories.path_outputs, "p1_aps.h5")), \
             'Processing failed (p1_aps.h5 not created).'
-        assert glob(os.path.join(config.data_directories.path_outputs, f"aps_parameters.h5")), \
-            f'Processing failed (aps_parameters.h5 not created).'
+        assert glob(os.path.join(config.data_directories.path_outputs, "aps_parameters.h5")), \
+            'Processing failed (aps_parameters.h5 not created).'
 
         # densification
         args.start = 4

From 36d60ed06f4a12fa7f9d7940422098eac221033b Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Tue, 6 Aug 2024 19:39:38 +0200
Subject: [PATCH 24/61] store APS method to aps_parameters.h5 in step 3 and
 apply filter in step 4 accordingly.

---
 sarvey/densification.py | 15 +++++++++------
 sarvey/filtering.py     |  2 +-
 sarvey/objects.py       | 13 ++++++++++---
 sarvey/processing.py    |  9 ++++++---
 4 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 28063181..a5191f62 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -527,24 +527,27 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     aps_params_obj = ApsParameters(file_path=aps_params_file, logger=logger)
     aps_params_obj.open()
 
-    if config.filtering.skip_filtering:
-        logger.info("Skip APS filtering")
+    filtering_method = aps_params_obj.method
+
+    if filtering_method == "None":
+        logger.info("Skip APS filtering. To enable APS filtering, update the config file and rerun step 3.")
         aps2_phase = np.zeros((point2_obj.num_points, point2_obj.ifg_net_obj.num_images))
     else:
-        logger.debug("Apply APS filtering to second-order points.")
-        if config.filtering.interpolation_method == "kriging":
+        logger.info("Apply APS filtering to second-order points.")
+
+        if filtering_method == "kriging":
             aps2_phase = applySpatialFilteringToP2(coord_utm1=aps1_obj.coord_utm,
                                                    residuals=aps_params_obj.phase,
                                                    coord_utm2=point2_obj.coord_utm,
                                                    model_name=aps_params_obj.model_name,
                                                    model_params=aps_params_obj.model_params,
                                                    logger=logger)
-        else:
+        elif filtering_method == "simple":
             aps2_phase = applySimpleInterpolationToP2(residuals=aps_params_obj.phase,
                                                       coord_utm1=aps1_obj.coord_utm,
                                                       coord_utm2=point2_obj.coord_utm,
                                                       logger=logger,
-                                                      interp_method=config.filtering.interpolation_method)
+                                                      interp_method=aps_params_obj.params_obj.model_name)
 
     p2_aps_file = join(output_path, f"coh{coh_value}_aps.h5")
     logger.info(f"Write APS for second-order points to file {p2_aps_file}.")
diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index de82f89b..88cee6b4 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -70,7 +70,7 @@ def applySpatialFilteringToP2(*, coord_utm1, residuals, coord_utm2, model_name,
     logger.debug(f"Start applying {num_time} time APS to {num_points2} second-order points using Kriging"
                  f" interpolation.")
 
-    if model_name == 'Stable':
+    if model_name == 'stable':
         logger.debug(f"Recreate {model_name} gs model using {model_params.shape[0]} parameters.")
         model = gs.Stable(dim=2)
         if model_params.shape[0] != 4:
diff --git a/sarvey/objects.py b/sarvey/objects.py
index fe732db7..a32dbf8b 100644
--- a/sarvey/objects.py
+++ b/sarvey/objects.py
@@ -809,6 +809,7 @@ def __init__(self, *, file_path: str, logger: Logger):
             Logging handler
         """
         self.file_path = file_path
+        self.method = None
         self.model_name = None
         self.model_params = None
         self.num_params = None
@@ -816,24 +817,28 @@ def __init__(self, *, file_path: str, logger: Logger):
         self.logger = logger
         self.logger.debug(f"ApsParameters initialized with file path: {file_path}")
 
-    def prepare(self, *, model_name: str = "Stable", model_params: np.ndarray, phase: np.ndarray):
+    def prepare(self, *, method: str = "kriging", model_name: str = "stable", model_params: np.ndarray,
+                phase: np.ndarray):
         """Prepare APS model parameters and store it into a file.
 
         Parameters
         ----------
+        method: str
+            method used for interpolation (kriging/simple)
         model_name: str
-            name of the model
+            name of the variogram model/interpolation type for kriging/simple
         model_params: np.ndarray
             Model parameters
         phase: np.ndarray
             Residual phase of P1 used for APS estimation
         """
-        self.logger.info(f"Preparing ApsParameters with model: {model_name}")
+        self.logger.info(f"Preparing ApsParameters with method {method} and model {model_name}")
 
         if model_params is None or model_params.size == 0:
             self.logger.debug("Model parameters are empty; defaulting to an empty list.")
             model_params = np.array([])
 
+        self.method = method
         self.model_name = model_name
         self.model_params = model_params
         self.num_params = model_params.shape[0] if model_params is not None and model_params.size > 0 else 0
@@ -855,6 +860,7 @@ def open(self):
             with h5py.File(self.file_path, 'r') as f:
                 self.model_params = f["model_params"][:]
                 self.phase = f["phase"][:]
+                self.method = f.attrs["method"]
                 self.model_name = f.attrs["model_name"]
                 self.num_params = f.attrs["num_params"]
                 self.logger.debug(f"Loaded model name: {self.model_name}, number of parameters: {self.num_params}")
@@ -873,6 +879,7 @@ def writeToFile(self):
             with h5py.File(self.file_path, 'w') as f:
                 f.create_dataset('model_params', data=self.model_params)
                 f.create_dataset('phase', data=self.phase)
+                f.attrs["method"] = self.method
                 f.attrs["model_name"] = self.model_name
                 f.attrs["num_params"] = self.num_params
 
diff --git a/sarvey/processing.py b/sarvey/processing.py
index e3d2061e..8ae05354 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -638,6 +638,7 @@ def runFiltering(self):
             msg += " SKIP ATMOSPHERIC FILTERING! "
             msg += "#" * 10
             self.logger.info(msg=msg)
+            method = "None"
             model_name = "None"
             aps_model_params = None
             num_points1 = phase_for_aps_filtering.shape[0]
@@ -646,7 +647,8 @@ def runFiltering(self):
         else:
             # spatial filtering of points with linear motion only (no non-linear motion)
             if self.config.filtering.interpolation_method == "kriging":
-                model_name = "Stable"  # To be integrated in the config?
+                method = "kriging"
+                model_name = "stable"  # To be integrated in the config?
                 aps1_phase, aps_model_params = estimateAtmosphericPhaseScreen(
                     residuals=phase_for_aps_filtering,
                     coord_utm1=point1_obj.coord_utm,
@@ -655,7 +657,8 @@ def runFiltering(self):
                     logger=self.logger
                 )
             else:
-                model_name = "Linear"  # To be integrated in the config?
+                method = "simple"
+                model_name = "linear"  # To be integrated in the config?
                 aps_model_params = None
                 aps1_phase = simpleInterpolation(
                     residuals=phase_for_aps_filtering,
@@ -667,7 +670,7 @@ def runFiltering(self):
         aps_file = join(self.path, "aps_parameters.h5")
         self.logger.debug(f"Prepare Aps Parameters file {aps_file}")
         aps_params_obj = ApsParameters(file_path=aps_file, logger=self.logger)
-        aps_params_obj.prepare(model_name=model_name, model_params=aps_model_params,
+        aps_params_obj.prepare(method=method, model_name=model_name, model_params=aps_model_params,
                                phase=phase_for_aps_filtering)
 
         point1_obj.phase -= aps1_phase

From 2a42710ca17f666297ee4123e0b67a85598f482f Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Thu, 17 Oct 2024 22:14:49 +0200
Subject: [PATCH 25/61] merge next_release into save-aps

---
 sarvey/config.py                | 35 ++++++++++---------------
 sarvey/densification.py         | 46 ++++++++++++++++-----------------
 sarvey/processing.py            |  6 ++---
 sarvey/sarvey_mti.py            |  9 +++----
 tests/test_processing.py        |  2 +-
 tests/testdata/config_test.json |  4 +--
 6 files changed, 47 insertions(+), 55 deletions(-)

diff --git a/sarvey/config.py b/sarvey/config.py
index 8aa01d43..bc94f701 100644
--- a/sarvey/config.py
+++ b/sarvey/config.py
@@ -485,12 +485,6 @@ class Filtering(BaseModel, extra=Extra.forbid):
         default=1000
     )
 
-    mask_p2_file: Optional[str] = Field(
-        title="Path to spatial mask file for second-order points.",
-        description="Path to the mask file, e.g. created by sarvey_mask.",
-        default=""
-    )
-
     use_moving_points: bool = Field(
         title="Use moving points",
         description="Set whether to use moving points in the filtering step.",
@@ -519,16 +513,6 @@ def checkGridSize(cls, v):
         else:
             return v
 
-    @validator('mask_p2_file')
-    def checkSpatialMaskPath(cls, v):
-        """Check if the path is correct."""
-        if v == "" or v is None:
-            return None
-        else:
-            if not os.path.exists(os.path.abspath(v)):
-                raise ValueError(f"mask_p2_file path is invalid: {v}")
-        return v
-
     @validator('max_temporal_autocorrelation')
     def checkMaxAutoCorr(cls, v):
         """Check if the value is correct."""
@@ -552,6 +536,12 @@ class Densification(BaseModel, extra=Extra.forbid):
         default=0.5
     )
 
+    mask_p2_file: Optional[str] = Field(
+        title="Path to spatial mask file for second-order points.",
+        description="Path to the mask file, e.g. created by sarvey_mask.",
+        default=""
+    )
+
     num_connections_p1: int = Field(
         title="Number of connections in temporal unwrapping.",
         description="Set number of connections between second-order point and closest first-order points for temporal "
@@ -599,11 +589,14 @@ def checkTempCohThrsh2(cls, v):
             raise ValueError("Temporal coherence threshold cannot be greater than 1.")
         return v
 
-    @validator('coherence_threshold')
-    def checkCoherenceThresh(cls, v):
-        """Check if coherence_threshold is valid."""
-        if v < 0 or v > 1:
-            raise ValueError(f"coherence_threshold is not between 0 and 1: {v}")
+    @validator('mask_p2_file')
+    def checkSpatialMaskPath(cls, v):
+        """Check if the path is correct."""
+        if v == "" or v is None:
+            return None
+        else:
+            if not os.path.exists(os.path.abspath(v)):
+                raise ValueError(f"mask_p2_file path is invalid: {v}")
         return v
 
     @validator('num_connections_p1')
diff --git a/sarvey/densification.py b/sarvey/densification.py
index 16ae950c..d8673ea8 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -309,7 +309,7 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     p1_file = join(output_path, "p1_ts_filt.h5")
     logger.debug(f"Create a mask based on points in file {p1_file}.")
     point1_obj = Points(file_path=p1_file, logger=logger)
-    point1_obj.open(path_inputs=config.data_directories.path_inputs)
+    point1_obj.open(input_path=config.general.input_path)
     p1_mask = point1_obj.createMask()  # used later for selecting psCand2 when a spatial mask AOI is given.
     logger.debug(f"Mask created with {p1_mask.sum()} selected points.")
 
@@ -329,9 +329,9 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     )  # first-order points are included in second-order points
     logger.debug(f"{cand_mask2.sum()} second-order points selected based on temporal coherence {coherence_p2:.2f}.")
 
-    if config.phase_linking.phase_linking:
+    if config.phase_linking.use_phase_linking_results:
         # read PL results
-        pl_file = join(config.phase_linking.path_inverted, "phase_series.h5")
+        pl_file = join(config.phase_linking.inverted_path, "phase_series.h5")
         logger.debug(f"Read phase linking results from file {pl_file}.")
         pl_coh = readfile.read(pl_file,
                                datasetName='temporalCoherence')[0]
@@ -340,8 +340,8 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
                                  datasetName='shp')[0]
 
         if config.phase_linking.use_ps:
-            logger.debug(f"Read phase linking PS mask from file {config.phase_linking.path_mask_file_ps}.")
-            mask_ps = readfile.read(config.phase_linking.path_mask_file_ps,
+            logger.debug(f"Read phase linking PS mask from file {config.phase_linking.mask_ps_file}.")
+            mask_ps = readfile.read(config.phase_linking.mask_ps_file,
                                     datasetName='mask')[0].astype(np.bool_)
             cand_mask_pl = (pl_coh > coherence_p2) | mask_ps
         else:
@@ -349,8 +349,8 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
             # remove ps candidates, because the ps detection strategy in miaplpy seems to be biased.
             cand_mask_pl[siblings <= config.phase_linking.num_siblings] = False
 
-        if config.phase_linking.spatial_mask_file_pl is not None:
-            path_mask_pl_aoi = join(config.phase_linking.spatial_mask_file_pl)
+        if config.phase_linking.mask_phase_linking_file is not None:
+            path_mask_pl_aoi = join(config.phase_linking.mask_phase_linking_file)
             logger.debug(f"Load mask for area of interest from file {path_mask_pl_aoi}.")
             mask_pl_aoi = readfile.read(path_mask_pl_aoi, datasetName='mask')[0].astype(np.bool_)
 
@@ -367,7 +367,7 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
             cbar.ax.set_visible(False)  # make size of axis consistent with all others
             plt.tight_layout()
             plt.title("Mask for phase linking points")
-            fig.savefig(join(output_path, "pic", "step_4_mask_coh{}_phase_linking.png".format(coh_value)), dpi=300)
+            fig.savefig(join(output_path, "pic", "step_4_mask_p2_coh{}_phase_linking.png".format(coh_value)), dpi=300)
             plt.close(fig)
 
             # mask points after plotting, so that available coherent points are visible in figure
@@ -380,8 +380,8 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
 
     mask_valid_area = ut.detectValidAreas(bmap_obj=bmap_obj, logger=logger)
 
-    if config.filtering.spatial_mask_file_p2 is not None:
-        path_mask_aoi = join(config.filtering.spatial_mask_file_p2)
+    if config.densification.mask_p2_file is not None:
+        path_mask_aoi = join(config.densification.mask_p2_file)
         logger.info(f"Load mask for area of interest from file {path_mask_aoi}.")
         mask_aoi = readfile.read(path_mask_aoi, datasetName='mask')[0].astype(np.bool_)
         mask_valid_area &= mask_aoi
@@ -407,10 +407,10 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     cbar.ax.set_visible(False)  # make size of axis consistent with all others
     plt.tight_layout()
     plt.title("Mask for dense point set")
-    fig.savefig(join(output_path, "pic", "step_4_mask_coh{}.png".format(coh_value)), dpi=300)
+    fig.savefig(join(output_path, "pic", "step_4_mask_p2_coh{}.png".format(coh_value)), dpi=300)
     plt.close(fig)
 
-    p2_file = join(output_path, f"coh{coh_value}_ifg_wr.h5")
+    p2_file = join(output_path, f"p2_coh{coh_value}_ifg_wr.h5")
     logger.debug(f"Prepare second-order points object to save to file {p2_file}.")
     point2_obj = Points(file_path=p2_file, logger=logger)
     coord_xy = np.array(np.where(cand_mask2)).transpose()
@@ -418,35 +418,35 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     point2_obj.prepare(
         point_id=point_id2,
         coord_xy=coord_xy,
-        path_inputs=config.data_directories.path_inputs
+        input_path=config.general.input_path
     )
 
     ifg_stack_file = join(output_path, "ifg_stack.h5")
     logger.debug(f"Read interferogram phase for selected second-order points from file {ifg_stack_file}.")
     ifg_stack_obj = BaseStack(file=ifg_stack_file, logger=logger)
     point2_obj.phase = ut.readPhasePatchwise(stack_obj=ifg_stack_obj, dataset_name="ifgs",
-                                             num_patches=config.processing.num_patches, cand_mask=cand_mask2,
+                                             num_patches=config.general.num_patches, cand_mask=cand_mask2,
                                              point_id_img=point_id_img, logger=logger)
 
-    if config.phase_linking.phase_linking:
-        pl_inverted_file = join(config.phase_linking.path_inverted, "phase_series.h5")
+    if config.phase_linking.use_phase_linking_results:
+        pl_inverted_file = join(config.phase_linking.inverted_path, "phase_series.h5")
         logger.debug(f"Read interferogram phase from MiaplPy results from file {pl_inverted_file}.")
         phase_linking_obj = BaseStack(file=pl_inverted_file,
                                       logger=logger)
 
         pl_phase = ut.readPhasePatchwise(
             stack_obj=phase_linking_obj, dataset_name="phase",
-            num_patches=config.processing.num_patches,
+            num_patches=config.general.num_patches,
             cand_mask=cand_mask2,
             point_id_img=point_id_img, logger=logger
         )
 
         # subset to time span
-        slc_stack_obj = slcStack(join(config.data_directories.path_inputs, "slcStack.h5"))
+        slc_stack_obj = slcStack(join(config.general.input_path, "slcStack.h5"))
         slc_stack_obj.open()
         time_mask = createTimeMaskFromDates(
             start_date=config.preparation.start_date,
-            stop_date=config.preparation.stop_date,
+            stop_date=config.preparation.end_date,
             date_list=slc_stack_obj.dateList,
             logger=logger
         )[0]
@@ -470,7 +470,7 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     p1_aps_file = join(output_path, "p1_aps.h5")
     logger.debug(f"Read APS for first order point from file {p1_aps_file}.")
     aps1_obj = Points(file_path=p1_aps_file, logger=logger)
-    aps1_obj.open(path_inputs=config.data_directories.path_inputs)
+    aps1_obj.open(input_path=config.general.input_path)
 
     aps_params_file = join(output_path, "aps_parameters.h5")
     logger.debug(f"Read APS parameters from file {aps_params_file}.")
@@ -499,12 +499,12 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
                                                       logger=logger,
                                                       interp_method=aps_params_obj.params_obj.model_name)
 
-    p2_aps_file = join(output_path, f"coh{coh_value}_aps.h5")
+    p2_aps_file = join(output_path, f"p2_coh{coh_value}_aps.h5")
     logger.info(f"Write APS for second-order points to file {p2_aps_file}.")
     aps2_obj = Points(file_path=p2_aps_file, logger=logger)
     aps2_obj.open(
-        other_file_path=join(output_path, f"coh{coh_value}_ifg_wr.h5"),
-        path_inputs=config.data_directories.path_inputs
+        other_file_path=join(output_path, f"p2_coh{coh_value}_ifg_wr.h5"),
+        input_path=config.general.input_path
     )
     aps2_obj.phase = aps2_phase
     aps2_obj.writeToFile()
diff --git a/sarvey/processing.py b/sarvey/processing.py
index cd92b7ec..7fe7d5e0 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -644,7 +644,7 @@ def runFiltering(self):
                 aps1_phase, aps_model_params = estimateAtmosphericPhaseScreen(
                     residuals=phase_for_aps_filtering,
                     coord_utm1=point1_obj.coord_utm,
-                    num_cores=self.config.processing.num_cores,
+                    num_cores=self.config.general.num_cores,
                     bool_plot=False,
                     logger=self.logger
                 )
@@ -708,7 +708,7 @@ def runDensificationTimeAndSpace(self):
         aps1_obj = Points(file_path=join(self.path, "p1_aps.h5"), logger=self.logger)
         aps1_obj.open(input_path=self.config.general.input_path)
 
-        if self.config.filtering.mask_p2_file is None:
+        if self.config.densification.mask_p2_file is None:
             """
             overview of points contained in the *_obj
             (unstable p1 means: p1 which were not used in atmospheric filtering)
@@ -797,7 +797,7 @@ def runDensificationTimeAndSpace(self):
             vel_p1=vel_p1,
             demerr_p1=demerr_p1,
             point2_obj=point2_obj,
-            num_conn_p1=self.config.densification.num_connections_to_p1,
+            num_conn_p1=self.config.densification.num_connections_p1,
             max_dist_p1=self.config.densification.max_distance_to_p1,
             velocity_bound=self.config.densification.velocity_bound,
             demerr_bound=self.config.densification.dem_error_bound,
diff --git a/sarvey/sarvey_mti.py b/sarvey/sarvey_mti.py
index af94d527..ee528ce2 100755
--- a/sarvey/sarvey_mti.py
+++ b/sarvey/sarvey_mti.py
@@ -147,8 +147,7 @@ def run(*, config: Config, args: argparse.Namespace, logger: Logger):
                            config_section_default=config_default_dict["filtering"],
                            logger=logger)
         proc_obj.runFiltering()
-    coh_value = int(config.filtering.coherence_p2 * 100)
-    required_files.extend(["p1_aps.h5", f"p2_coh{coh_value}_ifg_wr.h5", f"p2_coh{coh_value}_aps.h5"])
+    required_files.extend(["p1_aps.h5", "aps_parameters.h5"])
 
     if 4 in steps:
         checkIfRequiredFilesExist(
@@ -295,9 +294,9 @@ def main(iargs=None):
     if config.consistency_check.mask_p1_file is not None:
         config.consistency_check.mask_p1_file = os.path.abspath(
             join(args.workdir, config.consistency_check.mask_p1_file))
-    if config.filtering.mask_p2_file is not None:
-        config.filtering.mask_p2_file = os.path.abspath(
-            join(args.workdir, config.filtering.mask_p2_file))
+    if config.densification.mask_p2_file is not None:
+        config.densification.mask_p2_file = os.path.abspath(
+            join(args.workdir, config.densification.mask_p2_file))
 
     # create all necessary directories
     if not os.path.exists(config.general.output_path):
diff --git a/tests/test_processing.py b/tests/test_processing.py
index 0d14f9a1..7c8b87a4 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -320,7 +320,7 @@ def testMasking(self):
         config.general.apply_temporal_unwrapping = False
         config.preparation.ifg_network_type = "sb"
         config.consistency_check.mask_p1_file = "tests/testdata/aoi_mask.h5"
-        config.filtering.mask_p2_file = "tests/testdata/aoi_mask.h5"
+        config.densification.mask_p2_file = "tests/testdata/aoi_mask.h5"
 
         # preparation
         args.start = 0
diff --git a/tests/testdata/config_test.json b/tests/testdata/config_test.json
index faa0950f..e41e0b4d 100644
--- a/tests/testdata/config_test.json
+++ b/tests/testdata/config_test.json
@@ -44,14 +44,14 @@
         "apply_aps_filtering": true,
         "interpolation_method": "kriging",
         "grid_size": 1000,
-        "mask_p2_file": null,
         "use_moving_points": true,
         "max_temporal_autocorrelation": 0.3
     },
     "densification": {
         "coherence_p2": 0.9,
         "num_connections_p1": 5,
-        "max_distance_p1": 2000,
+        "mask_p2_file": null,
+        "max_distance_to_p1": 2000,
         "velocity_bound": 0.15,
         "dem_error_bound": 100.0,
         "num_optimization_samples": 100,

From 022e6d11e25aec1c5a9859cf2dcbab77af9cd778 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Fri, 18 Oct 2024 11:04:36 +0200
Subject: [PATCH 26/61] do not save aps2_obj in step 4

---
 sarvey/densification.py | 25 +++++++++++++------------
 sarvey/processing.py    | 28 ++++++++++++++++------------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index d8673ea8..6d8aead2 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -296,8 +296,8 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
     -------
     point2_obj : Points
         Points object with second-order points
-    aps2_obj: Points
-        Points object with APS estimation for second-order points
+    aps2_phase: np.ndarray
+        APS estimation for second-order points
     """
     # this function is mainly adapted from sarvey.processing.runFiltering
     # TODO: Directly pass input parameters instead of config dictionary
@@ -499,15 +499,16 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
                                                       logger=logger,
                                                       interp_method=aps_params_obj.params_obj.model_name)
 
-    p2_aps_file = join(output_path, f"p2_coh{coh_value}_aps.h5")
-    logger.info(f"Write APS for second-order points to file {p2_aps_file}.")
-    aps2_obj = Points(file_path=p2_aps_file, logger=logger)
-    aps2_obj.open(
-        other_file_path=join(output_path, f"p2_coh{coh_value}_ifg_wr.h5"),
-        input_path=config.general.input_path
-    )
-    aps2_obj.phase = aps2_phase
-    aps2_obj.writeToFile()
+    # TODO: delete this part after testing the aps estimation
+    # p2_aps_file = join(output_path, f"p2_coh{coh_value}_aps.h5")
+    # logger.info(f"Write APS for second-order points to file {p2_aps_file}.")
+    # aps2_obj = Points(file_path=p2_aps_file, logger=logger)
+    # aps2_obj.open(
+    #     other_file_path=join(output_path, f"p2_coh{coh_value}_ifg_wr.h5"),
+    #     input_path=config.general.input_path
+    # )
+    # aps2_obj.phase = aps2_phase
+    # aps2_obj.writeToFile()
 
     logger.debug("Finished selecting second-order points.")
-    return point2_obj, aps2_obj
+    return point2_obj, aps2_phase
diff --git a/sarvey/processing.py b/sarvey/processing.py
index 7fe7d5e0..9d460740 100644
--- a/sarvey/processing.py
+++ b/sarvey/processing.py
@@ -688,7 +688,7 @@ def runDensificationTimeAndSpace(self):
         coh_value = int(self.config.densification.coherence_p2 * 100)
         self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")
 
-        _, aps2_obj = selectP2(output_path=self.path, config=self.config, logger=self.logger)
+        _, aps2_phase = selectP2(output_path=self.path, config=self.config, logger=self.logger)
 
         point2_obj = Points(file_path=join(self.path, "p2_coh{}_ifg_unw.h5".format(coh_value)), logger=self.logger)
         point2_obj.open(
@@ -725,16 +725,16 @@ def runDensificationTimeAndSpace(self):
             mask_unstable_p1 = p1_mask & (~aps1_mask)
             unstable_p1_id = point_id_img[np.where(mask_unstable_p1)]
 
-            mask_unstable_p1_in_p2 = np.ones((aps2_obj.num_points,), dtype=np.bool_)
-            for p in aps2_obj.point_id:
+            mask_unstable_p1_in_p2 = np.ones((point2_obj.num_points,), dtype=np.bool_)
+            for p in point2_obj.point_id:
                 if p not in unstable_p1_id:
-                    mask_unstable_p1_in_p2[aps2_obj.point_id == p] = False
+                    mask_unstable_p1_in_p2[point2_obj.point_id == p] = False
 
             # add unstable p1 from aps2 to aps1
             aps1_obj.addPointsFromObj(
-                new_point_id=aps2_obj.point_id[mask_unstable_p1_in_p2],
-                new_coord_xy=aps2_obj.coord_xy[mask_unstable_p1_in_p2, :],
-                new_phase=aps2_obj.phase[mask_unstable_p1_in_p2, :],
+                new_point_id=point2_obj.point_id[mask_unstable_p1_in_p2],
+                new_coord_xy=point2_obj.coord_xy[mask_unstable_p1_in_p2, :],
+                new_phase=aps2_phase[mask_unstable_p1_in_p2, :],
                 new_num_points=mask_unstable_p1_in_p2[mask_unstable_p1_in_p2].shape[0],
                 input_path=self.config.general.input_path
             )
@@ -743,8 +743,10 @@ def runDensificationTimeAndSpace(self):
             p2_mask = point2_obj.createMask()
             mask_only_p2 = p2_mask & (~p1_mask)
             keep_id = point_id_img[np.where(mask_only_p2)]
+
+            mask = np.isin(point2_obj.point_id, keep_id)
+            aps2_phase = aps2_phase[mask, :]
             point2_obj.removePoints(keep_id=keep_id, input_path=self.config.general.input_path)
-            aps2_obj.removePoints(keep_id=keep_id, input_path=self.config.general.input_path)
 
         else:
             """
@@ -780,13 +782,15 @@ def runDensificationTimeAndSpace(self):
                 p2_mask = point2_obj.createMask()
                 mask_p2 = ~(p1_mask & p2_mask) & p2_mask
                 p2_id = point_id_img[np.where(mask_p2)]
+
+                mask = np.isin(point2_obj.point_id, p2_id)
+                aps2_phase = aps2_phase[mask, :]
                 point2_obj.removePoints(keep_id=p2_id, input_path=self.config.general.input_path)
-                aps2_obj.removePoints(keep_id=p2_id, input_path=self.config.general.input_path)
 
         # return to ifg-space
         a_ifg = point2_obj.ifg_net_obj.getDesignMatrix()
         aps1_ifg_phase = np.matmul(a_ifg, aps1_obj.phase.T).T
-        aps2_ifg_phase = np.matmul(a_ifg, aps2_obj.phase.T).T
+        aps2_ifg_phase = np.matmul(a_ifg, aps2_phase.T).T
 
         # correct for APS
         point2_obj.phase = np.angle(np.exp(1j * point2_obj.phase) * np.conjugate(np.exp(1j * aps2_ifg_phase)))
@@ -917,7 +921,7 @@ def runDensificationSpace(self):
         coh_value = int(coherence_p2 * 100)
 
         self.logger.info(f"Select second-order points using coherence threshold {coherence_p2:.2f}.")
-        _, aps2_obj = selectP2(output_path=self.path, config=self.config, logger=self.logger)
+        _, aps2_phase = selectP2(output_path=self.path, config=self.config, logger=self.logger)
 
         point_obj = Points(file_path=join(self.path, "p2_coh{}_ifg_unw.h5".format(coh_value)), logger=self.logger)
         point_obj.open(
@@ -927,7 +931,7 @@ def runDensificationSpace(self):
 
         # return to ifg-space
         a_ifg = point_obj.ifg_net_obj.getDesignMatrix()
-        aps2_ifg_phase = np.matmul(a_ifg, aps2_obj.phase.T).T
+        aps2_ifg_phase = np.matmul(a_ifg, aps2_phase.T).T
 
         # correct for APS
         point_obj.phase = np.angle(np.exp(1j * point_obj.phase) * np.conjugate(np.exp(1j * aps2_ifg_phase)))

From 4d48c7e65eccad5c3d4943fe829de35aaf5a76b9 Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sat, 26 Oct 2024 11:53:08 +0200
Subject: [PATCH 27/61] fix remaining utm to map coordinate after merging main
 into save-aps

---
 sarvey/densification.py | 8 ++++----
 sarvey/filtering.py     | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index d4e11597..aa6d0116 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -486,16 +486,16 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
         logger.info("Apply APS filtering to second-order points.")
 
         if filtering_method == "kriging":
-            aps2_phase = applySpatialFilteringToP2(coord_utm1=aps1_obj.coord_utm,
+            aps2_phase = applySpatialFilteringToP2(coord_map1=aps1_obj.coord_map,
                                                    residuals=aps_params_obj.phase,
-                                                   coord_utm2=point2_obj.coord_utm,
+                                                   coord_map2=point2_obj.coord_map,
                                                    model_name=aps_params_obj.model_name,
                                                    model_params=aps_params_obj.model_params,
                                                    logger=logger)
         elif filtering_method == "simple":
             aps2_phase = applySimpleInterpolationToP2(residuals=aps_params_obj.phase,
-                                                      coord_utm1=aps1_obj.coord_utm,
-                                                      coord_utm2=point2_obj.coord_utm,
+                                                      coord_map1=aps1_obj.coord_map,
+                                                      coord_map2=point2_obj.coord_map,
                                                       logger=logger,
                                                       interp_method=aps_params_obj.params_obj.model_name)
 
diff --git a/sarvey/filtering.py b/sarvey/filtering.py
index 8c18973d..ec465d13 100644
--- a/sarvey/filtering.py
+++ b/sarvey/filtering.py
@@ -190,7 +190,7 @@ def launchSpatialFiltering(parameters: tuple):
         atmospheric phase screen for the known points (size: num_points_p1 x num_ifgs)
     """
     # Unpack the parameters
-    (idx_range, num_time, residuals, coord_map1, coord_utm2, bins, bool_plot, logger) = parameters
+    (idx_range, num_time, residuals, coord_map1, bins, bool_plot, logger) = parameters
 
     x = coord_map1[:, 1]
     y = coord_map1[:, 0]

From 17e40b320ebb8fa2eacd023f5f5af8648364560e Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Sat, 26 Oct 2024 11:53:42 +0200
Subject: [PATCH 28/61] remove p2_coh*_aps.h5 in test.

---
 tests/test_processing.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/test_processing.py b/tests/test_processing.py
index fc2f99c0..5191cfc8 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -154,8 +154,6 @@ def testUnwrappingTimeAndSpace(self):
         coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_aps.h5")), \
-            f'Processing failed (p2_coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ts.h5")), \
@@ -228,8 +226,6 @@ def testUnwrappingSpace(self):
         coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_aps.h5")), \
-            f'Processing failed (p2_coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ts.h5")), \
@@ -304,8 +300,6 @@ def testPhaseLinking(self):
         coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_aps.h5")), \
-            f'Processing failed (p2_coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ts.h5")), \
@@ -380,8 +374,6 @@ def testMasking(self):
         coh_value = int(config.densification.coherence_p2 * 100)
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_wr.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_wr.h5 not created).'
-        assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_aps.h5")), \
-            f'Processing failed (p2_coh{coh_value}_aps.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ifg_unw.h5")), \
             f'Processing failed (p2_coh{coh_value}_ifg_unw.h5 not created).'
         assert glob(os.path.join(config.general.output_path, f"p2_coh{coh_value}_ts.h5")), \

From 3b7eba3d5026a8bd99b0d2b5d47e1d5e2e2a813c Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 15:33:59 +0100
Subject: [PATCH 29/61] Update ci.yml

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8af99a25..1ffa1c02 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -69,6 +69,7 @@ jobs:
           # wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/4b3be399dffa488e98db/?dl=1
           wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/104b499f6f7e4360877d/?dl=1          
           unzip testdata.zip
+          mkdir -p test
           mv testdata tests/
           mamba list
           make pytest

From 3dac33015a3afdc78d9d7166b9ffb0d5eb75d028 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 17:42:39 +0100
Subject: [PATCH 30/61] add pip installation in docs

---
 docs/installation.rst | 37 ++++++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index fb2d49e3..83ed46fd 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -13,10 +13,36 @@ SARvey is a cross-platform python-based software and can be installed on
 Linux
 -----
 
-On Linux, SARvey can be installed `Using Mamba (recommended)`_ or `Using Anaconda or Miniconda`_.
+On Linux, SARvey can be installed `Using Pip (recommended)`_ or `Using Mamba`_ or `Using Anaconda or Miniconda`_.
 
-Using Mamba (recommended)
-^^^^^^^^^^^^^^^^^^^^^^^^^
+Using pip (recommended)
+^^^^^^^^^^^^^^^^^^^^^^^
+Using pip_ (latest version recommended), **SARvey** is installed as follows:
+
+1. Create a new environment for **SARvey** (optional but recommended):
+   .. code-block:: bash
+
+    conda create -n sarvey pip -y 
+    conda activate sarvey
+
+2. Install **Miaplpy_**
+
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
+
+3. Install **SARvey**
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
+
+
+If your are a developer, use the following command to make sure all development requirements are installed.
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git#egg=sarvey[dev]
+    
+
+
+Using Mamba
+^^^^^^^^^^^
 
 Using mamba_ (latest version recommended), **SARvey** is installed as follows:
 
@@ -43,10 +69,6 @@ Using mamba_ (latest version recommended), **SARvey** is installed as follows:
     pip install .
 
 
-This is the preferred method to install **SARvey**, as it always installs the most recent stable release and
-automatically resolves all the dependencies.
-
-
 Using Anaconda or Miniconda
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -183,4 +205,5 @@ On Windows, SARvey is tested on Windows Subsystem for Linux (WSL_) version 2. Pl
 .. _mamba: https://github.com/mamba-org/mamba
 .. _Conda for Mac: https://docs.conda.io/projects/conda/en/latest/user-guide/install/macos.html
 .. _WSL: https://learn.microsoft.com/en-us/windows/wsl/
+.. _MiaplPy: https://github.com/insarlab/MiaplPy
 

From 798bd7de754b6b2aeae6df032764f0ef78d6cc63 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 17:45:50 +0100
Subject: [PATCH 31/61] add gdal to requirements in setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 8bc2c95f..a3c70066 100644
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,7 @@
 req = [
     "cython", "numpy", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
-    "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
+    "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri", "gdal"
 ]
 
 req_setup = []

From 9bcb47e9d130eb729f3a492bdb519b3fdfddb486 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:22:15 +0100
Subject: [PATCH 32/61] Update installation.rst to make mamba the recommended
 installation method

---
 docs/installation.rst | 58 +++++++++++++++++++++----------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 83ed46fd..955d8569 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -13,36 +13,10 @@ SARvey is a cross-platform python-based software and can be installed on
 Linux
 -----
 
-On Linux, SARvey can be installed `Using Pip (recommended)`_ or `Using Mamba`_ or `Using Anaconda or Miniconda`_.
+On Linux, SARvey can be installed `Using `Using Mamba (recommended)`_ or `Using Anaconda or Miniconda`_ or Pip`_.
 
-Using pip (recommended)
-^^^^^^^^^^^^^^^^^^^^^^^
-Using pip_ (latest version recommended), **SARvey** is installed as follows:
-
-1. Create a new environment for **SARvey** (optional but recommended):
-   .. code-block:: bash
-
-    conda create -n sarvey pip -y 
-    conda activate sarvey
-
-2. Install **Miaplpy_**
-
-    .. code-block:: bash
-     conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
-
-3. Install **SARvey**
-    .. code-block:: bash
-     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
-
-
-If your are a developer, use the following command to make sure all development requirements are installed.
-    .. code-block:: bash
-     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git#egg=sarvey[dev]
-    
-
-
-Using Mamba
-^^^^^^^^^^^
+Using Mamba (recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Using mamba_ (latest version recommended), **SARvey** is installed as follows:
 
@@ -97,6 +71,32 @@ Using conda_ (latest version recommended), **SARvey** is installed as follows:
     pip install .
 
 
+Using pip
+^^^^^^^^^
+
+Using pip_ (latest version recommended), **SARvey** is installed as follows:
+
+1. Create a new environment for **SARvey** (optional but recommended):
+   .. code-block:: bash
+
+    conda create -n sarvey pip -y 
+    conda activate sarvey
+
+2. Install **Miaplpy_**
+
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
+
+3. Install **SARvey**
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
+
+
+If your are a developer, use the following command to make sure all development requirements are installed.
+    .. code-block:: bash
+     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git#egg=sarvey[dev]
+
+
 MacOS ARM (Apple Silicon M2)
 ----------------------------
 

From 7e681caeba9d7cd0143cc03478a42be09ce13caf Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:22:46 +0100
Subject: [PATCH 33/61] Update installation.rst to fix typo

---
 docs/installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 955d8569..3c62b74c 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -13,7 +13,7 @@ SARvey is a cross-platform python-based software and can be installed on
 Linux
 -----
 
-On Linux, SARvey can be installed `Using `Using Mamba (recommended)`_ or `Using Anaconda or Miniconda`_ or Pip`_.
+On Linux, SARvey can be installed `Using Mamba (recommended)`_ or `Using Anaconda or Miniconda`_ or `Using Pip`_.
 
 Using Mamba (recommended)
 ^^^^^^^^^^^^^^^^^^^^^^^^^

From ce4e62cd841ca05c0d957206a827e15047651cb3 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:23:43 +0100
Subject: [PATCH 34/61] Update installation.rst

---
 docs/installation.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 3c62b74c..f73876fe 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -77,6 +77,7 @@ Using pip
 Using pip_ (latest version recommended), **SARvey** is installed as follows:
 
 1. Create a new environment for **SARvey** (optional but recommended):
+
    .. code-block:: bash
 
     conda create -n sarvey pip -y 
@@ -84,12 +85,13 @@ Using pip_ (latest version recommended), **SARvey** is installed as follows:
 
 2. Install **Miaplpy_**
 
-    .. code-block:: bash
-     conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
+   .. code-block:: bash
+    conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
 
 3. Install **SARvey**
-    .. code-block:: bash
-     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
+
+   .. code-block:: bash
+    conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
 
 
 If your are a developer, use the following command to make sure all development requirements are installed.

From e4bc4fba62228442e4bc84bf2ed2904579227e5c Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:35:57 +0100
Subject: [PATCH 35/61] add gdal to extra_req

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a3c70066..fb70ed99 100644
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,7 @@
 req = [
     "cython", "numpy", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
-    "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri", "gdal"
+    "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
 ]
 
 req_setup = []
@@ -63,6 +63,8 @@
 
 req_dev = ['twine'] + req_setup + req_test + req_doc + req_lint
 
+extra_req = ["gdal"]
+
 setup(
     author="Andreas Piter",
     author_email='piter@ipi.uni-hannover.de',

From fe51bed3d08bb6485cfa4c0e49e7cd6b02872fc5 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:43:48 +0100
Subject: [PATCH 36/61] numpy<2 in setup.py to avoid kaumi conflict

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index fb70ed99..66f386a3 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
     exec(version_file.read(), version)
 
 req = [
-    "cython", "numpy", "pyproj", "matplotlib", "numba", "scipy",
+    "cython", "numpy<2", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
     "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
 ]

From 6cd6fbfd4eea7c637496131d220ff0a533517559 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 18:53:06 +0100
Subject: [PATCH 37/61] Update setup.py to use numpy<=1.26

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 66f386a3..ca6f2f6d 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
     exec(version_file.read(), version)
 
 req = [
-    "cython", "numpy<2", "pyproj", "matplotlib", "numba", "scipy",
+    "cython", "numpy<=1.26", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
     "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
 ]

From 81bffa5b5c1c5df1bdc85326d1387f965b9d382f Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 19:13:18 +0100
Subject: [PATCH 38/61] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ca6f2f6d..fb70ed99 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
     exec(version_file.read(), version)
 
 req = [
-    "cython", "numpy<=1.26", "pyproj", "matplotlib", "numba", "scipy",
+    "cython", "numpy", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
     "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
 ]

From 5af61cce1e00e66a77e28e6d36eee10207ce1e8c Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 21:01:30 +0100
Subject: [PATCH 39/61] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index fb70ed99..ca6f2f6d 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
     exec(version_file.read(), version)
 
 req = [
-    "cython", "numpy", "pyproj", "matplotlib", "numba", "scipy",
+    "cython", "numpy<=1.26", "pyproj", "matplotlib", "numba", "scipy",
     "mintpy", "h5py", "overpy", "miaplpy", "gstools", "shapely", "pandas", "geopandas", "pymaxflow",
     "pillow", "pydantic<=1.10.10", "importlib_resources", "kamui", "json5", "cmcrameri"
 ]

From d053a31c225efea50fcb4d6b5b8bfac8844021c8 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 22:54:12 +0100
Subject: [PATCH 40/61] use ci_env directly and skip cloning to ci_tmp_env

---
 .github/workflows/ci.yml | 52 ++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1ffa1c02..7e698b00 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,28 +23,28 @@ jobs:
   before_script:
     runs-on: self-hosted
 
-    strategy:
-      matrix:
-        node-version: [ 20.x ]
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-
-      - name: Install dependencies
-        run: |
-          source /opt/conda/etc/profile.d/conda.sh
-          conda init bash
-          source ~/.bashrc
-          conda env remove --name ci_temp_env --yes || echo "Environment ci_temp_env does not exist"
-          conda create --prefix /home/runneruser/.conda/envs/ci_temp_env --clone ci_env
-          conda activate ci_temp_env
-        shell: bash
+    # strategy:
+    #   matrix:
+    #     node-version: [ 20.x ]
+
+    # steps:
+    #   - name: Checkout code
+    #     uses: actions/checkout@v4
+
+    #   - name: Set up Python
+    #     uses: actions/setup-python@v5
+    #     with:
+    #       python-version: '3.x'
+
+      # - name: Install dependencies
+      #   run: |
+      #     source /opt/conda/etc/profile.d/conda.sh
+      #     conda init bash
+      #     source ~/.bashrc
+      #     conda env remove --name ci_temp_env --yes || echo "Environment ci_temp_env does not exist"
+      #     conda create --prefix /home/runneruser/.conda/envs/ci_temp_env --clone ci_env
+      #     conda activate ci_temp_env
+      #   shell: bash
 
   test_sarvey:
     runs-on: self-hosted
@@ -64,7 +64,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          conda activate ci_temp_env
+          conda activate vi_env
           rm -rf tests/testdata
           # wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/4b3be399dffa488e98db/?dl=1
           wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/104b499f6f7e4360877d/?dl=1          
@@ -80,7 +80,7 @@ jobs:
         run: |
           conda init bash
           source ~/.bashrc
-          source activate ci_temp_env
+          source activate vi_env
           
           # replace documentation address for tags befro make docs
           IFS='/' read -r OWNER REPO <<< "${GITHUB_REPOSITORY}"
@@ -166,7 +166,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          conda activate ci_temp_env
+          conda activate vi_env
           make lint
         shell: bash
 
@@ -209,7 +209,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          source activate ci_temp_env
+          source activate vi_env
           make urlcheck
         shell: bash
 

From a7f23b161afd89477797fe8313e4527543a9c7bf Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 22:55:47 +0100
Subject: [PATCH 41/61] Update ci.yml

---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7e698b00..463445cd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,13 +23,13 @@ jobs:
   before_script:
     runs-on: self-hosted
 
-    # strategy:
-    #   matrix:
-    #     node-version: [ 20.x ]
+    strategy:
+      matrix:
+        node-version: [ 20.x ]
 
-    # steps:
-    #   - name: Checkout code
-    #     uses: actions/checkout@v4
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
 
     #   - name: Set up Python
     #     uses: actions/setup-python@v5

From 47acefe4173980f1ab0555854c36bb6cb656591a Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 22:59:35 +0100
Subject: [PATCH 42/61] fix typo

---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 463445cd..741ed35a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,7 +64,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          conda activate vi_env
+          conda activate ci_env
           rm -rf tests/testdata
           # wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/4b3be399dffa488e98db/?dl=1
           wget -nv -c -O testdata.zip https://seafile.projekt.uni-hannover.de/f/104b499f6f7e4360877d/?dl=1          
@@ -80,7 +80,7 @@ jobs:
         run: |
           conda init bash
           source ~/.bashrc
-          source activate vi_env
+          source activate ci_env
           
           # replace documentation address for tags befro make docs
           IFS='/' read -r OWNER REPO <<< "${GITHUB_REPOSITORY}"
@@ -166,7 +166,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          conda activate vi_env
+          conda activate ci_env
           make lint
         shell: bash
 
@@ -209,7 +209,7 @@ jobs:
           source /opt/conda/etc/profile.d/conda.sh
           conda init bash
           source ~/.bashrc
-          source activate vi_env
+          source activate ci_env
           make urlcheck
         shell: bash
 

From e3fca8525464b6e314f47b77b6bfab7ab1272015 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 23:02:38 +0100
Subject: [PATCH 43/61] remove before_script job

---
 .github/workflows/ci.yml | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 741ed35a..bd413d89 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,35 +20,8 @@ env:
     SKIP_TEST: false
 
 jobs:
-  before_script:
-    runs-on: self-hosted
-
-    strategy:
-      matrix:
-        node-version: [ 20.x ]
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-    #   - name: Set up Python
-    #     uses: actions/setup-python@v5
-    #     with:
-    #       python-version: '3.x'
-
-      # - name: Install dependencies
-      #   run: |
-      #     source /opt/conda/etc/profile.d/conda.sh
-      #     conda init bash
-      #     source ~/.bashrc
-      #     conda env remove --name ci_temp_env --yes || echo "Environment ci_temp_env does not exist"
-      #     conda create --prefix /home/runneruser/.conda/envs/ci_temp_env --clone ci_env
-      #     conda activate ci_temp_env
-      #   shell: bash
-
   test_sarvey:
     runs-on: self-hosted
-    needs: before_script
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

From 2c5eea83150988a2f386f8f80a063631272a9909 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 23:12:52 +0100
Subject: [PATCH 44/61] update test_install to replace conda with pip

---
 .github/workflows/ci.yml | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bd413d89..6f25bd10 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ permissions:
 
 env:
     SKIP_DEPLOY: false
-    SKIP_INSTALL: true
+    SKIP_INSTALL: false
     SKIP_TEST: false
 
 jobs:
@@ -202,18 +202,17 @@ jobs:
           run: |
               source /opt/conda/etc/profile.d/conda.sh
               conda activate base
-              mamba env remove --name sarvey_testinstall --yes || echo "Environment sarvey_testinstall does not exist"
-              mamba clean --index-cache --tarballs --packages -y
-              pip install conda-merge
-              export PATH=$HOME/.local/bin:$PATH # workaround conda-merge not found!
-              wget https://raw.githubusercontent.com/insarlab/MiaplPy/main/conda-env.yml
-              conda-merge conda-env.yml tests/CI_docker/context/environment_sarvey.yml > env.yml
-              mamba env create --name sarvey_testinstall -f env.yml
+              
+              conda create -n sarvey_testinstall python=3.10 pip  -y
               conda activate sarvey_testinstall
+              conda install conda-forge::pysolid -y
+              conda install conda-forge::gdal
               pip install git+https://github.com/insarlab/MiaplPy.git
-              pip install .
+              pip install git+https://github.com/mahmud1/sarvey.git@update-runner2
+              pip install sarvey[dev]
+
               OUTPUT=$(pip check) || { echo "$OUTPUT"; true; }
-              cd ..
+              conda list
               python -c "import sarvey; print(sarvey)"
           shell: bash
 

From 05ac8834e5af9b57d7e4d0971673575e0914e0c3 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 23:34:06 +0100
Subject: [PATCH 45/61] update docker file. move to root and rename
 environment.yml

---
 .../environment_sarvey.yml => environment.yml |  0
 .../CI_docker/build_sarvey_testsuite_image.sh |  1 -
 tests/CI_docker/context/entrypoint.sh         | 26 +++++++++++
 tests/CI_docker/context/sarvey_ci.docker      | 46 ++++++++++---------
 4 files changed, 51 insertions(+), 22 deletions(-)
 rename tests/CI_docker/context/environment_sarvey.yml => environment.yml (100%)
 create mode 100644 tests/CI_docker/context/entrypoint.sh

diff --git a/tests/CI_docker/context/environment_sarvey.yml b/environment.yml
similarity index 100%
rename from tests/CI_docker/context/environment_sarvey.yml
rename to environment.yml
diff --git a/tests/CI_docker/build_sarvey_testsuite_image.sh b/tests/CI_docker/build_sarvey_testsuite_image.sh
index c08ad9e2..04485541 100755
--- a/tests/CI_docker/build_sarvey_testsuite_image.sh
+++ b/tests/CI_docker/build_sarvey_testsuite_image.sh
@@ -12,7 +12,6 @@ print(version["__version__"])
 '
 version=`python -c "$python_script"`
 tag="sarvey_ci:$version"
-gitlab_runner="sarvey_gitlab_CI_runner"
 
 echo "#### Build runner docker image"
 if [[ "$(docker images ${tag} | grep ${tag} 2> /dev/null)" != "" ]]; then
diff --git a/tests/CI_docker/context/entrypoint.sh b/tests/CI_docker/context/entrypoint.sh
new file mode 100644
index 00000000..94325667
--- /dev/null
+++ b/tests/CI_docker/context/entrypoint.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+# Check for necessary environment variables
+if [ -z "$RUNNER_TOKEN" ] || [ -z "$RUNNER_REPO_URL" ]; then
+  echo "Error: RUNNER_TOKEN and RUNNER_REPO_URL environment variables must be set."
+  exit 1
+fi
+
+# Configure the runner if it hasn’t been configured already
+if [ ! -f ".runner" ]; then
+  echo "Configuring the GitHub Actions runner..."
+  ./config.sh --unattended \
+    --replace \
+    --labels self-hosted \
+    --url "$RUNNER_REPO_URL" \
+    --token "$RUNNER_TOKEN" \
+    --name "${RUNNER_NAME:-docker-runner}" || exit 1
+fi
+
+# Trap SIGTERM and deregister the runner on shutdown if needed
+trap './config.sh remove --token "$RUNNER_TOKEN"; exit 0' SIGTERM
+
+echo "Starting the GitHub Actions runner..."
+exec ./run.sh
+
diff --git a/tests/CI_docker/context/sarvey_ci.docker b/tests/CI_docker/context/sarvey_ci.docker
index 51c4a85b..d34dcc45 100644
--- a/tests/CI_docker/context/sarvey_ci.docker
+++ b/tests/CI_docker/context/sarvey_ci.docker
@@ -1,5 +1,8 @@
 FROM condaforge/miniforge3:latest
 
+# Set Mamba root prefix
+ENV MAMBA_ROOT_PREFIX="/opt/conda"
+
 # update base environment
 RUN --mount=type=cache,target=/opt/conda/pkgs \
     mamba update --all -y && \
@@ -8,22 +11,16 @@ RUN --mount=type=cache,target=/opt/conda/pkgs \
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN mkdir actions-runner; cd actions-runner && \
-    apt-get update && apt-get install -y curl
+    apt-get update && apt-get install -y curl gfortran build-essential openssh-client
 
 WORKDIR /actions-runner
 
-RUN curl -o actions-runner-linux-x64-2.317.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz && \
-    tar xzf ./actions-runner-linux-x64-2.317.0.tar.gz && \
+RUN curl -o actions-runner-linux-x64.tar.gz -L  https://github.com/actions/runner/releases/download/v2.322.0/actions-runner-linux-x64-2.322.0.tar.gz&& \
+    tar xzf ./actions-runner-linux-x64.tar.gz && \
     ./bin/installdependencies.sh && \
     useradd -m runneruser && \
     chown -R runneruser:runneruser /actions-runner
 
-USER runneruser
-
-RUN  ./config.sh --url https://github.com/luhipi/sarvey --token <TOKEN>  --unattended --replace --name mefe2_sarvey_ci_1.0.0 --labels self-hosted
-
-USER root
-
 # install some needed packages
 RUN --mount=type=cache,target=/opt/conda/pkgs \
     mamba install -y bzip2 fish gcc gdb git ipython make nano pip tree wget unzip
@@ -31,22 +28,29 @@ RUN --mount=type=cache,target=/opt/conda/pkgs \
 # use bash shell instead of sh shell for all docker commands
 SHELL ["/bin/bash", "-c"]
 
-# copy some needed stuff to /root
-COPY ../environment_sarvey.yml .
 
-# create ci_env environment
+# Create ci_env environment with pip installed
 RUN --mount=type=cache,target=/opt/conda/pkgs \
-    pip install conda-merge && \
-    wget https://raw.githubusercontent.com/insarlab/MiaplPy/main/conda-env.yml && \
-    conda-merge conda-env.yml environment_sarvey.yml > env.yml && \
-    mamba env create -n ci_env -f env.yml && \
-    source /opt/conda/bin/activate ci_env && \
-    pip install git+https://github.com/insarlab/MiaplPy.git && \
-    conda list && \
+    conda create -n ci_env python=3.10 pip  -y && \
     conda clean -afy
 
-USER runneruser
+# Install additional packages using mamba and pip
+RUN --mount=type=cache,target=/opt/conda/pkgs \
+    conda install -n ci_env conda-forge::pysolid -y && \
+    conda install -n ci_env conda-forge::gdal && \
+    conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git && \
+    # TODO: replace the followong with the main branch
+    #conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git@main && \
+    conda run -n ci_env pip install git+https://github.com/mahmud1/sarvey.git@update-runner2 && \
+    conda run -n ci_env pip install sarvey[dev] && \
+    conda run -n ci_env pip install sphinx_rtd_theme && \
+    conda clean -afy
 
+COPY ../entrypoint.sh .
+RUN chown runneruser:runneruser entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+USER runneruser
 RUN chmod +x /actions-runner/run.sh
 
-ENTRYPOINT ["/actions-runner/run.sh"]
+ENTRYPOINT ["./entrypoint.sh"]

From 855d2849d96d48fdaa2df4d4e8fdba7bb8d3f89d Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 18 Feb 2025 23:43:16 +0100
Subject: [PATCH 46/61] update installation documentation with environment.yml
 in root.

---
 docs/installation.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index f73876fe..7dcdc2f2 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -21,7 +21,7 @@ Using Mamba (recommended)
 Using mamba_ (latest version recommended), **SARvey** is installed as follows:
 
 
-1. Clone the SARvey source code and install SARvey and all dependencies from the environment_sarvey.yml file:
+1. Clone the SARvey source code and install SARvey and all dependencies from the environment.yml file:
 
    .. code-block:: bash
 
@@ -35,7 +35,7 @@ Using mamba_ (latest version recommended), **SARvey** is installed as follows:
 
     pip install conda-merge
     wget https://raw.githubusercontent.com/insarlab/MiaplPy/main/conda-env.yml
-    conda-merge conda-env.yml tests/CI_docker/context/environment_sarvey.yml > env.yml
+    conda-merge conda-env.yml environment.yml > env.yml
     mamba env create -n sarvey -f env.yml
     rm env.yml conda-env.yml
     mamba activate sarvey
@@ -49,7 +49,7 @@ Using Anaconda or Miniconda
 Using conda_ (latest version recommended), **SARvey** is installed as follows:
 
 
-1. Then clone the **SARvey** source code and install **SARvey** and all dependencies from the environment_sarvey.yml file:
+1. Then clone the **SARvey** source code and install **SARvey** and all dependencies from the environment.yml file:
 
    .. code-block:: bash
 
@@ -63,7 +63,7 @@ Using conda_ (latest version recommended), **SARvey** is installed as follows:
 
     pip install conda-merge
     wget https://raw.githubusercontent.com/insarlab/MiaplPy/main/conda-env.yml
-    conda-merge conda-env.yml tests/CI_docker/context/environment_sarvey.yml > env.yml
+    conda-merge conda-env.yml environment.yml > env.yml
     conda env create -n sarvey -f env.yml
     rm env.yml conda-env.yml
     conda activate sarvey
@@ -144,12 +144,12 @@ Using conda_ (latest version recommended), SARvey is installed as follows:
         git clone https://github.com/luhipi/sarvey.git
         cd sarvey
 
-   2.2 Open `tests/CI_docker/context/environment_sarvey.yml` in an editor of your choice and comment out the lines `isce2` and `gcc_linux-64`. Alternatively, you can run the following commands.
+   2.2 Open `environment.yml` in an editor of your choice and comment out the lines `isce2` and `gcc_linux-64`. Alternatively, you can run the following commands.
 
     .. code-block:: bash
 
-         sed -i '' '/isce2/s/^/# /' tests/CI_docker/context/environment_sarvey.yml
-         sed -i '' '/gcc_linux-64/s/^/# /' tests/CI_docker/context/environment_sarvey.yml
+         sed -i '' '/isce2/s/^/# /' environment.yml
+         sed -i '' '/gcc_linux-64/s/^/# /' environment.yml
 
     Note: As of the time of creation of this document, `isce2` for MacOS ARM64 is not available in Conda repositories. Therefore, it is skipped, but it should not cause any problems for running SARvey. Also, `gcc_linux-64` is not required on ARM64.
 
@@ -157,7 +157,7 @@ Using conda_ (latest version recommended), SARvey is installed as follows:
 
     .. code-block:: bash
 
-        conda env update --name sarvey -f tests/CI_docker/context/environment_sarvey.yml
+        conda env update --name sarvey -f environment.yml
         conda activate sarvey
         pip install .
 

From 028e4cd8658f66c92523ec58cb04ac933a058297 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Wed, 19 Feb 2025 14:39:46 +0100
Subject: [PATCH 47/61] Update installation.rst

---
 docs/installation.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 7dcdc2f2..27d70121 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -83,19 +83,23 @@ Using pip_ (latest version recommended), **SARvey** is installed as follows:
     conda create -n sarvey pip -y 
     conda activate sarvey
 
-2. Install **Miaplpy_**
+2. Install **Miaplpy**
 
    .. code-block:: bash
+
     conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
 
 3. Install **SARvey**
 
    .. code-block:: bash
+
     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
 
 
 If your are a developer, use the following command to make sure all development requirements are installed.
+
     .. code-block:: bash
+
      conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git#egg=sarvey[dev]
 
 

From a370ae946229f69123ad5148ebe03cd34a24e3fc Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Thu, 20 Feb 2025 15:04:33 +0100
Subject: [PATCH 48/61] update pip installation in docs

---
 docs/installation.rst | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 27d70121..57dc57ad 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -83,25 +83,30 @@ Using pip_ (latest version recommended), **SARvey** is installed as follows:
     conda create -n sarvey pip -y 
     conda activate sarvey
 
-2. Install **Miaplpy**
+2. Install dependencies
 
    .. code-block:: bash
 
-    conda run -n ci_env pip install git+https://github.com/insarlab/MiaplPy.git
+    conda install conda-forge::pysolid
+    conda install conda-forge::pysolid
 
-3. Install **SARvey**
+
+3. Install **Miaplpy**
 
    .. code-block:: bash
 
-    conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git
+    pip install git+https://github.com/insarlab/MiaplPy.git
 
+4. Install **SARvey**
 
-If your are a developer, use the following command to make sure all development requirements are installed.
+   .. code-block:: bash
+
+    pip install git+https://github.com/luhipi/sarvey.git
 
-    .. code-block:: bash
 
-     conda run -n ci_env pip install git+https://github.com/luhipi/sarvey.git#egg=sarvey[dev]
+If your are a developer, use the following command to make sure all development requirements are installed.
 
+    #### TODO: add dev installation.
 
 MacOS ARM (Apple Silicon M2)
 ----------------------------

From 3d194fe9a8cf1a7fa3a1a9a95bf5fffa2d5d2559 Mon Sep 17 00:00:00 2001
From: piter <piter@ipi.uni-hannover.de>
Date: Mon, 24 Feb 2025 16:45:05 +0100
Subject: [PATCH 49/61] Fix numerical problems when computing the grid size.

---
 sarvey/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sarvey/utils.py b/sarvey/utils.py
index b811719b..2702d218 100644
--- a/sarvey/utils.py
+++ b/sarvey/utils.py
@@ -430,8 +430,8 @@ def splitImageIntoBoxesRngAz(*, length: int, width: int, num_box_az: int, num_bo
     num_box: int
         number of boxes
     """
-    y_step = int(np.rint((length / num_box_rng) / 10) * 10)
-    x_step = int(np.rint((width / num_box_az) / 10) * 10)
+    y_step = int(length / num_box_rng)
+    x_step = int(width / num_box_az)
 
     box_list = []
     y0 = 0

From 462de88be001295f4d4eac4f09a9cb4f11e7b784 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Thu, 27 Feb 2025 14:31:47 +0100
Subject: [PATCH 50/61] Create .readthedocs.yaml

---
 .readthedocs.yaml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 .readthedocs.yaml

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..e152e07b
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,32 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+    # You can also specify other tool versions:
+    # nodejs: "23"
+    # rust: "1.82"
+    # golang: "1.23"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#    install:
+#    - requirements: docs/requirements.txt

From 3e650a373956d1f0ada66ab61c1c6716180e69f7 Mon Sep 17 00:00:00 2001
From: piter <piter@ipi.uni-hannover.de>
Date: Fri, 28 Feb 2025 09:46:46 +0100
Subject: [PATCH 51/61] Update history.

---
 HISTORY.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/HISTORY.rst b/HISTORY.rst
index adebb52d..2f3d89af 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -6,6 +6,7 @@ History
 Future minor version (release soon)
 -----------------------------------
 
+* Fix numerical problems when computing grid size.
 
 1.2.0 (2025-02-19)
 ------------------

From a9313b2628f2a85ca044ec992381bb200783062d Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Mon, 3 Mar 2025 17:27:00 +0100
Subject: [PATCH 52/61] Update README.rst

---
 README.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/README.rst b/README.rst
index 6d14f21a..f0c184f7 100644
--- a/README.rst
+++ b/README.rst
@@ -32,10 +32,6 @@ Status
         :target: https://doi.org/10.5281/zenodo.12544130
         :alt: DOI
 
-
-See also the latest coverage_ report and the pytest_ HTML report.
-
-
 License
 -------
 
@@ -191,8 +187,6 @@ This package was created with Cookiecutter_ and the `fernlab/cookiecutter-python
 
 .. _Cookiecutter: https://github.com/audreyr/cookiecutter
 .. _`fernlab/cookiecutter-python-package`: https://git.gfz-potsdam.de/fernlab/products/cookiecutters/cookiecutter-python-package
-.. _coverage: https://luhipi.github.io/sarvey/main/coverage/
-.. _pytest: https://luhipi.github.io/sarvey/main/test_reports/report.html
 .. _processing: https://luhipi.github.io/sarvey/main/docs/processing.html
 .. _`processing steps`: https://luhipi.github.io/sarvey/main/docs/processing.html#processing-steps-for-two-step-unwrapping-workflow
 .. _preparation: https://luhipi.github.io/sarvey/main/docs/preparation.html

From 41b430ef6095070afe15488665a1de3977135462 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Mon, 3 Mar 2025 17:29:14 +0100
Subject: [PATCH 53/61] skip deployment in ci.yml

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8af99a25..310e1128 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ permissions:
   contents: write
 
 env:
-    SKIP_DEPLOY: false
+    SKIP_DEPLOY: true
     SKIP_INSTALL: true
     SKIP_TEST: false
 

From 05ff3cb5800500a2aa5062aa8efb0b9bf24c50ad Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 4 Mar 2025 10:46:46 +0100
Subject: [PATCH 54/61] update link to docs in README.rst

---
 README.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index f0c184f7..f0c2243f 100644
--- a/README.rst
+++ b/README.rst
@@ -10,7 +10,7 @@ Open-source InSAR time series analysis software developed within the project SAR
 Documentation
 -------------
 The documentation with installation instructions, processing steps, and examples with a demo dataset can be found at:
-https://luhipi.github.io/sarvey/main/docs/
+https://luhipi.github.io/sarvey/main
 
 Discussion
 ----------
@@ -26,7 +26,7 @@ Status
         :target: https://github.com/luhipi/sarvey/actions
         :alt: Pipelines
 .. image:: https://img.shields.io/static/v1?label=Documentation&message=GitHub%20Pages&color=blue
-        :target: https://luhipi.github.io/sarvey/main/docs/
+        :target: https://luhipi.github.io/sarvey/main
         :alt: Documentation
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.12544130.svg
         :target: https://doi.org/10.5281/zenodo.12544130
@@ -187,13 +187,13 @@ This package was created with Cookiecutter_ and the `fernlab/cookiecutter-python
 
 .. _Cookiecutter: https://github.com/audreyr/cookiecutter
 .. _`fernlab/cookiecutter-python-package`: https://git.gfz-potsdam.de/fernlab/products/cookiecutters/cookiecutter-python-package
-.. _processing: https://luhipi.github.io/sarvey/main/docs/processing.html
-.. _`processing steps`: https://luhipi.github.io/sarvey/main/docs/processing.html#processing-steps-for-two-step-unwrapping-workflow
-.. _preparation: https://luhipi.github.io/sarvey/main/docs/preparation.html
-.. _`Two-step unwrapping`: https://luhipi.github.io/sarvey/main/docs/processing.html#processing-steps-for-two-step-unwrapping-workflow
-.. _`One-step unwrapping`: https://luhipi.github.io/sarvey/main/docs/processing.html#processing-steps-for-one-step-unwrapping-workflow
-.. _`installation instruction`: https://luhipi.github.io/sarvey/main/docs/installation.html
-.. _`history`: https://luhipi.github.io/sarvey/main/docs/history.html
+.. _processing: https://luhipi.github.io/sarvey/main/processing.html
+.. _`processing steps`: https://luhipi.github.io/sarvey/main/processing.html#processing-steps-for-two-step-unwrapping-workflow
+.. _preparation: https://luhipi.github.io/sarvey/main/preparation.html
+.. _`Two-step unwrapping`: https://luhipi.github.io/sarvey/main/processing.html#processing-steps-for-two-step-unwrapping-workflow
+.. _`One-step unwrapping`: https://luhipi.github.io/sarvey/main/processing.html#processing-steps-for-one-step-unwrapping-workflow
+.. _`installation instruction`: https://luhipi.github.io/sarvey/main/installation.html
+.. _`history`: https://luhipi.github.io/sarvey/main/history.html
 .. _MiaplPy: https://github.com/insarlab/MiaplPy
 .. _MintPy: https://github.com/insarlab/MintPy
 .. _ISCE: https://github.com/isce-framework/isce2

From cee1ca781b72f2ffdd0a778f2ee4d6b6f2a28c6b Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Tue, 4 Mar 2025 12:31:04 +0100
Subject: [PATCH 55/61] Revert "Create .readthedocs.yaml"

This reverts commit 462de88be001295f4d4eac4f09a9cb4f11e7b784.
---
 .readthedocs.yaml | 32 --------------------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 .readthedocs.yaml

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
deleted file mode 100644
index e152e07b..00000000
--- a/.readthedocs.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# .readthedocs.yaml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-24.04
-  tools:
-    python: "3.13"
-    # You can also specify other tool versions:
-    # nodejs: "23"
-    # rust: "1.82"
-    # golang: "1.23"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-
-# Optionally build your docs in additional formats such as PDF and ePub
-# formats:
-#    - pdf
-#    - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#    install:
-#    - requirements: docs/requirements.txt

From c4b3f96f87b96ce506c654e42c1dc4e857b5b933 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 4 Mar 2025 13:02:12 +0100
Subject: [PATCH 56/61] update dev installation in docs

---
 docs/installation.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 57dc57ad..c451e2de 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -104,9 +104,12 @@ Using pip_ (latest version recommended), **SARvey** is installed as follows:
     pip install git+https://github.com/luhipi/sarvey.git
 
 
-If your are a developer, use the following command to make sure all development requirements are installed.
+If your are a developer, install the development requirements using the following command.
+
+   .. code-block:: bash
+
+    pip install sarvey[dev]
 
-    #### TODO: add dev installation.
 
 MacOS ARM (Apple Silicon M2)
 ----------------------------

From f1ce49991d460c94d1bb57b30e2d63f1c8c8478b Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 4 Mar 2025 16:01:53 +0100
Subject: [PATCH 57/61] Update ci.yml to test install via pip instead of mamba

---
 .github/workflows/ci.yml | 171 ++++++++++++++++++++++-----------------
 1 file changed, 97 insertions(+), 74 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f25bd10..a0184d63 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - main
+      - 'v?*b'
+      - 'v?*beta'
   push:
     branches:
       - main
@@ -15,7 +17,7 @@ permissions:
   contents: write
 
 env:
-    SKIP_DEPLOY: false
+    SKIP_DEPLOY: true
     SKIP_INSTALL: false
     SKIP_TEST: false
 
@@ -208,7 +210,28 @@ jobs:
               conda install conda-forge::pysolid -y
               conda install conda-forge::gdal
               pip install git+https://github.com/insarlab/MiaplPy.git
-              pip install git+https://github.com/mahmud1/sarvey.git@update-runner2
+              
+              # get current branch for installation
+              IFS='/' read -r OWNER REPO <<< "${GITHUB_REPOSITORY}"
+                            
+              URL="https://github.com/${OWNER}/${REPO}"
+              URL_IO="https://${OWNER}.github.io/${REPO}"
+              
+              echo "Repository URL: ${URL}"
+              echo "Repository Documentation URL: ${URL}"
+                            
+              if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+                  current_ref="${GITHUB_REF##*/}"
+              elif [[ "$GITHUB_REF" == refs/heads/* ]]; then
+                  current_ref="${GITHUB_REF#refs/heads/}"
+              elif [[ "$GITHUB_REF" == refs/pull/* ]]; then
+                  current_ref="${GITHUB_HEAD_REF}"
+              else
+                  current_ref="${GITHUB_REF}"
+              fi
+              
+              echo "Current branch/tag: ${URL}.git@$current_ref"
+              pip install git+${URL}.git@$current_ref
               pip install sarvey[dev]
 
               OUTPUT=$(pip check) || { echo "$OUTPUT"; true; }
@@ -216,77 +239,77 @@ jobs:
               python -c "import sarvey; print(sarvey)"
           shell: bash
 
-  deploy_pages:
-    runs-on: self-hosted
-
-    needs: 
-      - test_sarvey
-      - test_urls
-      - test_styles
-    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-
-      - name: Download docs
-        uses: actions/download-artifact@v4
-        with:
-          name: docs
-          path: docs/_build/html/
-
-      - name: Download coverage report
-        uses: actions/download-artifact@v4
-        with:
-          name: coverage-report
-          path: htmlcov/
-
-      - name: Download report.html
-        uses: actions/download-artifact@v4
-        with:
-          name: test-report
-
-      - name: Deploy to GitHub Pages
-        # trigger if merged into the main branch || published new tag
-        if: env.SKIP_DEPLOY == 'false' && github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
-        run: |
-          rm -rf public
+  # deploy_pages:
+  #   runs-on: self-hosted
+
+  #   needs: 
+  #     - test_sarvey
+  #     - test_urls
+  #     - test_styles
+  #   if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.x'
+
+  #     - name: Download docs
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: docs
+  #         path: docs/_build/html/
+
+  #     - name: Download coverage report
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: coverage-report
+  #         path: htmlcov/
+
+  #     - name: Download report.html
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: test-report
+
+  #     - name: Deploy to GitHub Pages
+  #       # trigger if merged into the main branch || published new tag
+  #       if: env.SKIP_DEPLOY == 'false' && github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+  #       run: |
+  #         rm -rf public
           
-          git clone --branch gh-pages https://github.com/${{ github.repository }} public
-
-          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
-            TAG_NAME=${GITHUB_REF##*/}
-            echo "Deploying to GitHub Pages for version tag: $TAG_NAME"
-            DOCS_PATH=public/tags/$TAG_NAME
-          else
-            echo "Deploying to GitHub Pages for main branch"
-            DOCS_PATH=public/main
-          fi
-
-          rm -rf  $DOCS_PATH
-          mkdir -p $DOCS_PATH/docs
-          mkdir -p $DOCS_PATH/images
-          mkdir -p $DOCS_PATH/coverage
-          mkdir -p $DOCS_PATH/test_reports
+  #         git clone --branch gh-pages https://github.com/${{ github.repository }} public
+
+  #         if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+  #           TAG_NAME=${GITHUB_REF##*/}
+  #           echo "Deploying to GitHub Pages for version tag: $TAG_NAME"
+  #           DOCS_PATH=public/tags/$TAG_NAME
+  #         else
+  #           echo "Deploying to GitHub Pages for main branch"
+  #           DOCS_PATH=public/main
+  #         fi
+
+  #         rm -rf  $DOCS_PATH
+  #         mkdir -p $DOCS_PATH/docs
+  #         mkdir -p $DOCS_PATH/images
+  #         mkdir -p $DOCS_PATH/coverage
+  #         mkdir -p $DOCS_PATH/test_reports
           
-          cp -r docs/_build/html/* $DOCS_PATH/docs
-          cp -r htmlcov/* $DOCS_PATH/coverage/
-          cp report.html $DOCS_PATH/test_reports/
-
-          ls -al $DOCS_PATH
-          ls -al $DOCS_PATH/docs
-          ls -al $DOCS_PATH/coverage
-          ls -al $DOCS_PATH/test_reports
-
-        shell: bash
-
-      - name: Upload to GitHub Pages
-        if: env.SKIP_DEPLOY == 'false'
-        uses: peaceiris/actions-gh-pages@v4
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./public
+  #         cp -r docs/_build/html/* $DOCS_PATH/docs
+  #         cp -r htmlcov/* $DOCS_PATH/coverage/
+  #         cp report.html $DOCS_PATH/test_reports/
+
+  #         ls -al $DOCS_PATH
+  #         ls -al $DOCS_PATH/docs
+  #         ls -al $DOCS_PATH/coverage
+  #         ls -al $DOCS_PATH/test_reports
+
+  #       shell: bash
+
+  #     - name: Upload to GitHub Pages
+  #       if: env.SKIP_DEPLOY == 'false'
+  #       uses: peaceiris/actions-gh-pages@v4
+  #       with:
+  #         github_token: ${{ secrets.GITHUB_TOKEN }}
+  #         publish_dir: ./public

From ea77b34eb5810c95f3788c483433039e53818949 Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Tue, 4 Mar 2025 17:06:10 +0100
Subject: [PATCH 58/61] Update installation.rst

---
 docs/installation.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index c451e2de..9e50d8f2 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -87,8 +87,7 @@ Using pip_ (latest version recommended), **SARvey** is installed as follows:
 
    .. code-block:: bash
 
-    conda install conda-forge::pysolid
-    conda install conda-forge::pysolid
+    conda install -c conda-forge pysolid gdal
 
 
 3. Install **Miaplpy**

From 77e5f3cd8996e7c53e6859c5f12d8c22851e10ef Mon Sep 17 00:00:00 2001
From: Mahmud Haghighi <mahmud1@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:34:05 +0100
Subject: [PATCH 59/61] Update HISTORY.rst

---
 HISTORY.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/HISTORY.rst b/HISTORY.rst
index 2f3d89af..210661b4 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -6,6 +6,9 @@ History
 Future minor version (release soon)
 -----------------------------------
 
+* Update CI docker builder
+* Update runner to test installation
+* Update documentation with new instruction for installation including pip
 * Fix numerical problems when computing grid size.
 
 1.2.0 (2025-02-19)

From 022053aae7bafd93eb3c72880bf5ba03776fcbdd Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Mon, 31 Mar 2025 16:55:56 +0200
Subject: [PATCH 60/61] merge main into save-aps

---
 sarvey/densification.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/sarvey/densification.py b/sarvey/densification.py
index 22607388..cc64ecf3 100644
--- a/sarvey/densification.py
+++ b/sarvey/densification.py
@@ -35,10 +35,12 @@
 from scipy.spatial import KDTree
 from logging import Logger
 import matplotlib.pyplot as plt
+import cmcrameri as cmc
 
 from mintpy.utils import ptime, readfile
 from miaplpy.objects.slcStack import slcStack
 
+
 from sarvey.unwrapping import oneDimSearchTemporalCoherence
 from sarvey.objects import Points, AmplitudeImage, BaseStack, ApsParameters
 import sarvey.utils as ut
@@ -356,12 +358,12 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
 
             fig = plt.figure(figsize=(15, 5))
             ax = fig.add_subplot()
-            ax.imshow(mask_pl_aoi, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
+            ax.imshow(mask_pl_aoi, cmap=cmc.cm.cmaps["grayC"], alpha=0.5, zorder=10, vmin=0, vmax=1)
             bmap_obj.plot(ax=ax, logger=logger)
             coord_xy = np.array(np.where(cand_mask_pl)).transpose()
             val = np.ones_like(cand_mask_pl)
             sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask_pl], s=0.5,
-                            cmap=plt.get_cmap("autumn_r"),
+                            cmap=cmc.cm.cmaps["lajolla_r"],
                             vmin=1, vmax=2)  # set min, max to ensure that points are yellow
             cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
             cbar.ax.set_visible(False)  # make size of axis consistent with all others
@@ -397,12 +399,12 @@ def selectP2(*, output_path: str, config: Config, logger: Logger):
 
     fig = plt.figure(figsize=(15, 5))
     ax = fig.add_subplot()
-    ax.imshow(mask_valid_area, cmap=plt.cm.get_cmap("gray"), alpha=0.5, zorder=10, vmin=0, vmax=1)
+    ax.imshow(mask_valid_area, cmap=cmc.cm.cmaps["grayC"], alpha=0.5, zorder=10, vmin=0, vmax=1)
     bmap_obj.plot(ax=ax, logger=logger)
     coord_xy = np.array(np.where(cand_mask2)).transpose()
     val = np.ones_like(cand_mask2)
-    sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask2], s=0.5, cmap=plt.get_cmap("autumn_r"),
-                    vmin=1, vmax=2)  # set min, max to ensure that points are yellow
+    sc = ax.scatter(coord_xy[:, 1], coord_xy[:, 0], c=val[cand_mask2], s=0.5, cmap=cmc.cm.cmaps["lajolla_r"],
+                    vmin=0, vmax=10)  # set min, max to ensure that points are yellow
     cbar = plt.colorbar(sc, pad=0.03, shrink=0.5)
     cbar.ax.set_visible(False)  # make size of axis consistent with all others
     plt.tight_layout()

From 5996f991469c5d6ded1a0f345f082324448c837b Mon Sep 17 00:00:00 2001
From: mahmud1 <mahmud1@users.noreply.github.com>
Date: Mon, 31 Mar 2025 17:35:52 +0200
Subject: [PATCH 61/61] merge main into save-aps

---
 sarvey/config.py                | 8 ++++----
 tests/testdata/config_test.json | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sarvey/config.py b/sarvey/config.py
index aa19f2b6..9405989d 100644
--- a/sarvey/config.py
+++ b/sarvey/config.py
@@ -542,7 +542,7 @@ class Densification(BaseModel, extra=Extra.forbid):
         default=""
     )
 
-    num_connections_p1: int = Field(
+    num_connections_to_p1: int = Field(
         title="Number of connections in temporal unwrapping.",
         description="Set number of connections between second-order point and closest first-order points for temporal "
                     "unwrapping.",
@@ -599,11 +599,11 @@ def checkSpatialMaskPath(cls, v):
                 raise ValueError(f"mask_p2_file path is invalid: {v}")
         return v
 
-    @validator('num_connections_p1')
+    @validator('num_connections_to_p1')
     def checkNumConn1(cls, v):
-        """Check if num_connections_p1 are valid."""
+        """Check if num_connections_to_p1 are valid."""
         if v <= 0:
-            raise ValueError(f"num_connections_p1 must be greater than 0: {v}")
+            raise ValueError(f"num_connections_to_p1 must be greater than 0: {v}")
         return v
 
     @validator('max_distance_to_p1')
diff --git a/tests/testdata/config_test.json b/tests/testdata/config_test.json
index e41e0b4d..4a5b916a 100644
--- a/tests/testdata/config_test.json
+++ b/tests/testdata/config_test.json
@@ -49,7 +49,7 @@
     },
     "densification": {
         "coherence_p2": 0.9,
-        "num_connections_p1": 5,
+        "num_connections_to_p1": 5,
         "mask_p2_file": null,
         "max_distance_to_p1": 2000,
         "velocity_bound": 0.15,