Merge pull request #142 from asmith75218/post_i09_update

Pre-P04 update

Author: asmith75218

ctdcal/cfg.yaml

@@ -1,6 +1,6 @@
 # Example user configuration file
 #
-# To use, call ctdcal.common.load_user_config(<this file>)
+# To use, call ctdcal.common.load_user_config(this_file)
 
 # Application Settings
 # --------------------
@@ -25,119 +25,82 @@ bottleflags_man: bottle_flags_manual.json
 # of different types, it is recommended to use a different
 # configuration file (this file) for each.
 
-# Sample frequency (Hz): int
-freq: 24
+inst_params:
+    # Sample frequency (Hz): int
+    freq: 24
+    # Number of water sampler bottles: int
+    max_bottles: 36
+
+# Fitting Settings
+# ----------------
+
+# Fit groups
+#   To define a new group for a parameter, enter a new line with
+#   the starting cast id (cast name) for the group.
+fit_groups:
+    pressure:
+        - '00101'
+    conductivity:
+        - '00101'
+    temperature:
+        - '00101'
+    rinko:
+        - '00101'
+
+# Fit coeffs
+#   ctdcal will use these values during calibration/fitting stages
+#   for primary and secondary channel sensors, based on the fit
+#   groups defined above. When additional fit groups are defined,
+#   add corresponding lines here for prim_n and secn_n.
+fit_coeffs:
+    temperature:
+        primary:
+            - primary_0: {P_order: 1, T_order: 0, zRange: "1000:6000"}
+#             - primary_n: {P_order: 1, T_order: 0, zRange: "1000:6000"}
+        secondary:
+            - secondary_0: {P_order: 1, T_order: 0, zRange: "1000:6000"}
+#             - secondary_n: {P_order: 1, T_order: 0, zRange: "1000:6000"}
+    conductivity:
+        primary:
+            - primary_0: {P_order: 1, T_order: 0, C_order: 0, zRange: "1000:6000"}
+#            - primary_1: {P_order: 1, T_order: 0, C_order: 0, zRange: "1000:6000"}
+        secondary:
+            - secondary_0: {P_order: 1, T_order: 0, C_order: 0, zRange: "1000:6000"}
+#            - secondary_1: {P_order: 1, T_order: 0, C_order: 0, zRange: "1000:6000"}
 
 # Advanced Settings
 # -----------------
 # These settings are provided for fine-tuning of specific CTDCAL
 # features. These should not normally need to change.
 
-# Conductivity threshold (in cast units): int or float
-#   Used to determine if the instrument is operating in the water
-cond_threshold: 20
-# Soak threshold (in pressure units): int or float
-#   The deepest depth at which the pre-cast soak may occur
-soak_threshold: 20
-# Soak rolling average window (seconds): int
-soak_win: 20
-# Filter window (seconds): int
-#   Used by the cast smoothing filter
-filter_win: 2
-# Filter type: boxcar, hann, triangle
-filter_type: hann
-# Columns to filter:
-filter_cols:
-  - CTDPRS
-  - CTDTMP1
-  - CTDTMP2
-  - CTDCOND1
-  - CTDCOND2
-  - CTDSAL
-  - U_DEF_poly1
-#  - FREE1
-  - CTDOXYVOLTS
-#  - REFTMP
-#  - GPSLAT
-#  - GPSLON
-
-# Legacy Settings
-# ---------------
-# Included for transitional compatibility with legacy configs.
-# Any NEW CODE that uses these is not guaranteed to work in the future!
-# TODO: move, eliminate or redefine these as appropriate
-
-expocode: ""
-section_id: ""
-ctd_serial: ""
-
-# Exchange-format export cols:
-ctd_col_names:
-  - CTDPRS
-  - CTDTMP1
-  - CTDTMP1_FLAG_W
-  - CTDSAL1
-  - CTDSAL1_FLAG_W
-  - CTDOXY1
-  - CTDOXY1_FLAG_W
-#  - CTDRINKO
-#  - CTDRINKO_FLAG_W
-  - CTDXMISS1
-  - CTDXMISS1_FLAG_W
-  - CTDFLUOR1
-  - CTDFLUOR1_FLAG_W
-
-ctd_col_units:
-  - DBAR
-  - ITS-90
-  - ""
-  - PSS-78
-  - ""
-  - UMOL/KG
-  - ""
-#  - UMOL/KG
-#  - ""
-  - 0-5VDC
-  - ""
-  - 0-5VDC
-  - ""
-
-# Legacy data directory structure, incompatible with user-configured datadir
-# above. Use with caution.
-dirs:
-  ssscc: "data/ssscc/"
-  raw: "data/raw/"
-  converted: "data/converted/"
-  time: "data/time/"
-  pressure: "data/pressure/"
-  bottle: "data/bottle/"
-  salt: "data/salt/"
-  reft: "data/reft/"
-  oxygen: "data/oxygen/"
-  logs: "data/logs/"
-  flags: "data/flags/"
-
-fig_dirs:
-    t1: "data/logs/fitting_figs/temp_primary/"
-    t2: "data/logs/fitting_figs/temp_secondary/"
-    c1: "data/logs/fitting_figs/cond_primary/"
-    c2: "data/logs/fitting_figs/cond_secondary/"
-    ox: "data/logs/fitting_figs/oxy_primary/"
-    rinko: "data/logs/fitting_figs/oxy_rinko/"
+# Soak parameters - used by ctd soak detection algorithm
+soak_det_params:
+    # Conductivity threshold (in cast units): int or float
+    #   Used to determine if the instrument is operating in the water
+    cond_threshold: 20
+    # Soak threshold (in pressure units): int or float
+    #   The deepest depth at which the pre-cast soak may occur
+    soak_threshold: 20
+    # Soak rolling average window (seconds): int
+    soak_win: 20
 
-# Legacy column name lookup
-column:
-    p: "CTDPRS"
-    t1: "CTDTMP1"
-    t2: "CTDTMP2"
-    c1: "CTDCOND1"
-    c2: "CTDCOND2"
-    sal: "CTDSAL"
-    rinko_oxy: "U_DEF_poly1"
-    oxyvolts: "CTDOXYVOLTS"
-    refT: "REFTMP"
-    refC: "BTLCOND"
-    refS: "SALNTY"
-    refO: "OXYGEN"
-    lat: "GPSLAT"
-    lon: "GPSLON"
+# Filter parameters - used by the cast smoothing filter
+filter_params:
+    # Filter window (seconds): int
+    filter_win: 2
+    # Filter type: boxcar, hann, triangle
+    filter_type: hann
+    # Columns to filter:
+    filter_cols:
+      - CTDPRS
+      - CTDTMP1
+      - CTDTMP2
+      - CTDCOND1
+      - CTDCOND2
+      - CTDSAL
+      - U_DEF_poly1
+    #  - FREE1
+      - CTDOXYVOLTS
+    #  - REFTMP
+    #  - GPSLAT
+    #  - GPSLON

ctdcal/common.py

@@ -20,6 +20,10 @@
 cfg = get_ctdcal_config()
 
 
+# Constants
+# ---------
+
+
 # Function definitions
 # --------------------
 
@@ -38,6 +42,7 @@ def load_user_config(cfgfile):
     -------
     Munch object
     """
+    cfgfile = validate_file(cfgfile)
     with open(cfgfile, 'r') as f:
         cfg = yaml.safe_load(f)
         return munchify(cfg)
@@ -202,6 +207,22 @@ def load_exchange_btl(btl_file: Union[str, Path]) -> pd.DataFrame:
     )
 
 
+# File exports
+def list_to_file(fname, outdir, lst):
+    """
+    Writes a list to a file, one list item to a line.
+
+    Parameters
+    ----------
+    fname : str or Path-like
+    outdir : str or Path-like
+    lst : iterable
+    """
+    outdir = validate_dir(outdir)
+    outfile = Path(outdir, fname)
+    with open(outfile, 'w') as f:
+        f.write('\n'.join(lst))
+
 def load_exchange_ctd(
     ctd_file: Union[str, Path, BufferedIOBase],
     n_files=None,
@@ -300,17 +321,74 @@ def load_exchange_ctd(
     )
 
 
-## Cast list manipulation
-def make_ssscc_list(fname="data/ssscc.csv"):
+## Cast list functions
+def make_cast_id_list(rawdir, outdir=None, pattern='*.hex', fname='cast_id_list.csv'):
     """
     Attempt to automatically generate list of station/casts from raw files.
     """
-    raw_files = Path(cfg.dirs["raw"]).glob("*.hex")
-    ssscc_list = sorted([f.stem for f in raw_files])
-    pd.Series(ssscc_list, dtype=str).to_csv(fname, header=None, index=False, mode="x")
+    search_dir = validate_dir(Path(rawdir))
+    raw_files = Path(search_dir).glob(pattern)
+    cast_id_list = sorted([f.stem for f in raw_files])
+    if len(cast_id_list) < 1:
+        raise FileNotFoundError('No raw data files found.')
+    # pd.Series(casts, dtype=str).to_csv(fname, header=None, index=False, mode="x")
+    if outdir is not None:
+        outdir = validate_dir(outdir, create=True)
+        list_to_file(fname, outdir, cast_id_list)
+    return cast_id_list
+
+
+def load_fit_groups(casts, fit_groups):
+    fit_groups_all = dict()
+    for name, fit_group in fit_groups.items():
+        for value in fit_group:
+            ct = casts.count(value)
+            if ct != 1:
+                raise ValueError("Invalid fit group definitions: %s must occur only once, but %s occurrances were found."
+                                 % (value, ct))
+        group_casts = []
+        for starting_cast in fit_group:
+            start_idx = casts.index(starting_cast)
+            if starting_cast == fit_group[-1]:
+                end_idx = None
+            else:
+                next_cast = fit_group[fit_group.index(starting_cast) + 1]
+                end_idx = casts.index(next_cast)
+            group_casts.append([cast for cast in casts[start_idx: end_idx]])
+        fit_groups_all[name] = group_casts
+    return munchify(fit_groups_all)
+
+def get_cast_id_list(fname, rawdir, outdir, auto_generate=True):
+    """
+    Loads a cast list from a file. If the file is not found, the cast list will be
+    auto-generated from a directory of raw files when auto_generate is True (default
+    behavior).
 
-    return ssscc_list
+    Parameters
+    ----------
+    fname
+    auto_generate
+
+    Returns
+    -------
+
+    """
+    fname = Path(outdir, fname)
+    try:
+        # test if file exists
+        fname = validate_file(fname)
+    except FileNotFoundError:
+        # auto-generate or die
+        if auto_generate is True:
+            make_cast_id_list(rawdir, outdir, fname=fname)
+        else:
+            raise
 
+    # file exists or has been auto-generated
+    with open(fname, "r") as f:
+        # skip comment lines
+        id_list = [line.strip() for line in f.readlines() if not line.startswith('#')]
+    return id_list
 
 def get_ssscc_list(fname="data/ssscc.csv"):
     """
@@ -327,6 +405,7 @@ def get_ssscc_list(fname="data/ssscc.csv"):
     list
         Cast names or identifiers, as a list of strings.
     """
+    log.warning("Use of get_ssscc_list() is deprecated. Use get_cast_id_list() instead.")
     ssscc_list = []
     with open(fname, "r") as lines:
         for line in lines:

ctdcal/fitting/fit_common.py

@@ -70,6 +70,28 @@ class NodeNotFoundError(Exception):
 
 # BottleFlag wrangling
 # TODO: Move to ctdcal.flagging.common after reorg
+def bf_from_json(fname):
+    """
+    Reads a JSON file and returns the contents as a BottleFlags object.
+
+    Parameters
+    ----------
+    fname : str or Path-like
+
+    Returns
+    -------
+    BottleFlags object
+    """
+    with open(fname, 'r') as f:
+        buf = f.read()
+    if buf == '':
+        # File exists but is empty
+        flags = BottleFlags()
+    else:
+        flags = BottleFlags.fromJSON(buf)
+    return flags
+
+
 def df_node_to_BottleFlags(df):
     """
     Convert a flag node from a DataFrame to a formatted BottleFlags object
@@ -129,13 +151,7 @@ def save_node(fname, node, label, create_new=False):
         If true, create a new node if it does not exist. Default is false.
 
     """
-    with open(fname, 'r') as f:
-        buf = f.read()
-    if buf == '':
-        # File exists but is empty
-        flags = BottleFlags()
-    else:
-        flags = BottleFlags.fromJSON(buf)
+    flags = bf_from_json(fname)
     if label in flags or create_new is True:
         flags[label] = node
     else: