Merge branch 'main' of https://github.com/Capstone-AutoML/AutoML_Capstone

Author: Arc-Celt

docker-compose.yml

@@ -27,11 +27,20 @@ services:
     volumes:
       - .:/app
 
-  human_intervention:
+  auto_labeling:
     image: celt313/automl_capstone:v0.0.3
     profiles: ["optional"]
     entrypoint: bash
-    command: -c "source activate capstone_env && python src/pipeline/human_intervention.py"
+    command: -c "source activate capstone_env && ./fetch_dataset.sh && python src/label_main.py"
+    volumes:
+      - .:/app
+
+
+  train_compress:
+    image: celt313/automl_capstone:v0.0.3
+    profiles: ["optional"]
+    entrypoint: bash
+    command: -c "source activate capstone_env && python src/train_compress.py"
     volumes:
       - .:/app
 

docs/_modules/index.html

@@ -49,7 +49,7 @@
 <li class="toctree-l1"><a class="reference internal" href="../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../quantization.html">Quantization</a></li>

docs/_modules/src/pipeline/augmentation.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../api_reference/index.html">Code Reference</a></li>
@@ -96,7 +97,32 @@ <h1>Source code for src.pipeline.augmentation</h1><div class="highlight"><pre>
 <div class="viewcode-block" id="build_augmentation_transform">
 <a class="viewcode-back" href="../../../api_reference/augmentation.html#src.pipeline.augmentation.build_augmentation_transform">[docs]</a>
 <span class="k">def</span><span class="w"> </span><span class="nf">build_augmentation_transform</span><span class="p">(</span><span class="n">config</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">A</span><span class="o">.</span><span class="n">Compose</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Build the augmentation transform pipeline from config.&quot;&quot;&quot;</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Build the augmentation transform pipeline from the given configuration.</span>
+
+<span class="sd">    This function constructs an Albumentations Compose object with a sequence of </span>
+<span class="sd">    image augmentation transforms. Each transform is applied with a configurable </span>
+<span class="sd">    probability and parameter set drawn from the `config` dictionary.</span>
+
+<span class="sd">    Supported transforms:</span>
+<span class="sd">    - HorizontalFlip</span>
+<span class="sd">    - RandomBrightnessContrast</span>
+<span class="sd">    - HueSaturationValue</span>
+<span class="sd">    - Blur</span>
+<span class="sd">    - GaussNoise</span>
+<span class="sd">    - ToGray</span>
+<span class="sd">    - Rotate</span>
+
+<span class="sd">    The transform also ensures bounding box alignment using &#39;pascal_voc&#39; format.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        config (dict): Dictionary containing probability values and parameters </span>
+<span class="sd">                       for each augmentation transform.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        A.Compose: An Albumentations Compose object with the specified transformations </span>
+<span class="sd">                   and bounding box handling.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="n">A</span><span class="o">.</span><span class="n">Compose</span><span class="p">(</span>
         <span class="p">[</span>
         <span class="c1"># Flips image horizontally, applied 50% of the time by default</span>
@@ -132,8 +158,27 @@ <h1>Source code for src.pipeline.augmentation</h1><div class="highlight"><pre>
                    <span class="n">num_augmentations</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
                    <span class="n">config</span><span class="p">:</span> <span class="nb">dict</span>
     <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Applies augmentations to each image N times and saves results.</span>
-<span class="sd">    Also saves un-augmented images with no predictions into separate folders.</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Apply augmentations to each labeled image and save the results.</span>
+
+<span class="sd">    For each (image, label) pair, this function applies the given transformation </span>
+<span class="sd">    pipeline `num_augmentations` times. It saves the augmented images and their </span>
+<span class="sd">    updated prediction labels (in JSON format) to the specified output directories.</span>
+
+<span class="sd">    If an image has no predictions (empty bounding box list), the original image </span>
+<span class="sd">    is saved separately in a dedicated &#39;no_prediction_images&#39; folder.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        matched_pairs (list): List of tuples, each containing a Path to a JSON file </span>
+<span class="sd">                              and its corresponding image file.</span>
+<span class="sd">        transform (A.Compose): Albumentations transformation pipeline.</span>
+<span class="sd">        output_img_dir (Path): Directory to save augmented images.</span>
+<span class="sd">        output_json_dir (Path): Directory to save augmented label files.</span>
+<span class="sd">        num_augmentations (int): Number of times to apply augmentations per image.</span>
+<span class="sd">        config (dict): Configuration dictionary that may include a base random seed.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        None</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="c1"># Separate folder for un-augmented no-prediction images and labels</span>
     <span class="n">no_pred_img_dir</span> <span class="o">=</span> <span class="n">output_img_dir</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">&quot;no_prediction_images&quot;</span>
@@ -201,14 +246,31 @@ <h1>Source code for src.pipeline.augmentation</h1><div class="highlight"><pre>
 <a class="viewcode-back" href="../../../api_reference/augmentation.html#src.pipeline.augmentation.augment_dataset">[docs]</a>
 <span class="k">def</span><span class="w"> </span><span class="nf">augment_dataset</span><span class="p">(</span><span class="n">image_dir</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">output_dir</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">config</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Orchestrates augmentation pipeline.</span>
-<span class="sd">    image_dir: path to labeled images</span>
-<span class="sd">    output_dir: root output directory (will contain &#39;images&#39; and &#39;labels&#39;)</span>
-<span class="sd">    config: dictionary of augmentation settings</span>
+<span class="sd">    Orchestrates the full augmentation pipeline for a labeled image dataset.</span>
+
+<span class="sd">    This function matches labeled JSON files with their corresponding images,</span>
+<span class="sd">    builds the augmentation pipeline from the provided config, and applies</span>
+<span class="sd">    augmentations using `augment_images`.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        image_dir (Path): Directory containing the original labeled images.</span>
+<span class="sd">        output_dir (Path): Root directory where augmented &#39;images/&#39; and &#39;labels/&#39; will be saved.</span>
+<span class="sd">        config (dict): Dictionary containing augmentation settings, including</span>
+<span class="sd">                       number of augmentations and optional transform parameters.</span>
+
+<span class="sd">    Behavior:</span>
+<span class="sd">        - Loads label files from a `labeled_json_dir` (‘automl_workspace/data_pipeline/labeled’)</span>
+<span class="sd">        - Matches JSON labels to image files by filename stem</span>
+<span class="sd">        - Builds an Albumentations transform pipeline using `build_augmentation_transform`</span>
+<span class="sd">        - Applies the transform using `augment_images` with `num_augmentations` per image</span>
+<span class="sd">        - Logs counts of label files, image files, and successful matches</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        None</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">num_augmentations</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;num_augmentations&quot;</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
 
-    <span class="n">labeled_json_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">&quot;automl_workspace/data_pipeline/labeled&quot;</span><span class="p">)</span>
+    <span class="n">labeled_json_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;label_dir&quot;</span><span class="p">,</span> <span class="s2">&quot;automl_workspace/data_pipeline/labeled&quot;</span><span class="p">))</span>
     <span class="n">output_img_dir</span> <span class="o">=</span> <span class="n">output_dir</span> <span class="o">/</span> <span class="s2">&quot;images&quot;</span>
     <span class="n">output_json_dir</span> <span class="o">=</span> <span class="n">output_dir</span> <span class="o">/</span> <span class="s2">&quot;labels&quot;</span>
 

docs/_modules/src/pipeline/clean_pipeline.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../api_reference/index.html">Code Reference</a></li>

docs/_modules/src/pipeline/distillation/distillation.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../api_reference/index.html">Code Reference</a></li>

docs/_modules/src/pipeline/human_intervention.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../api_reference/index.html">Code Reference</a></li>

docs/_modules/src/pipeline/prelabelling/grounding_dino_prelabelling.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../api_reference/index.html">Code Reference</a></li>

docs/_modules/src/pipeline/prelabelling/matching.html

@@ -49,10 +49,11 @@
 <li class="toctree-l1"><a class="reference internal" href="../../../../gdino_prelabelling.html">Grounding DINO Prelabeling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../matching_logic.html">Matching Logic</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../human_in_loop.html">Human-in-the-Loop</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation Module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../augmentation.html">Augmentation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../training.html">Training</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../distillation.html">Distillation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../clean_pipeline.html">File Cleaning and Archiving</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../tests/index.html">Unit Tests</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../database.html">Database Schema Overview</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../api_reference/index.html">Code Reference</a></li>