PIPELINE_DETAILED.md

flowchart TD
    %% ===== STAGE 0: Format & Normalize =====
    subgraph S0["0. Format & Normalize"]
        direction TB
        S0_00["0.0_optional_format_counts_file.ipynb"]
        S0_01["0.1_Remove_covariates_keepbmiadiposcore.R"]
        S0_02["0.2_create_peak_file.ipynb"]
        S0_AGG[("Normalized BED + peaks")]
        S0_00 --> S0_01 --> S0_02 --> S0_AGG
    end

    %% ===== STAGE 1: Call CMs =====
    subgraph S1["1. Call CMs"]
        direction TB
        subgraph S1_PHM["PHM"]
            S1_P0["1.0_phm_data_preparation.ipynb"]
            S1_P1["1.1_run_phm.sh"]
            S1_P0 --> S1_P1
        end
        subgraph S1_CLO["Clomics"]
            S1_C1["1.1_run_clomics.sh"]
        end
        subgraph S1_VCM["VCMtools"]
            S1_V1["1.1_run_vcmtools.sh"]
        end
        S1_AGG[("CM outputs<br/>(all methods)")]
        S1_P1 --> S1_AGG
        S1_C1 --> S1_AGG
        S1_V1 --> S1_AGG
    end

    S0_AGG --> S1_P0
    S0_AGG --> S1_C1
    S0_AGG --> S1_V1

    %% ===== STAGE 2: Compare CMs =====
    subgraph S2["2. Compare CMs"]
        direction TB
        subgraph S2_ORIG["scripts/original"]
            S2_O1["2.0.1_prepare_cm_annotation_files_per_method.ipynb"]
            S2_O2["2.0.2_get_peak_overlaps_per_method.ipynb"]
            S2_O3["2.0.3_compare_methods_per_cell_type_test_data.sh"]
            S2_O4["2.1_nCMs_lengths_sizes_by_timepoint.py"]
            S2_O5["2.2_get_CM_similarities_across_methods_heatmap_venn_upset.ipynb"]
            S2_O1 --> S2_O2 --> S2_O3
            S2_O3 --> S2_O4
            S2_O3 --> S2_O5
        end
        subgraph S2_MOD["scripts/bulk_vs_sc"]
            S2_M1["2.0.1_prepare_cm_annotation_files_per_modality.ipynb"]
            S2_M2["2.0.2_get_peak_overlaps_per_modality.ipynb"]
            S2_M3["2.0.3_compare_modalities_per_method.sh"]
            S2_M4["2.1_nCMs_lengths_sizes_by_modality.py"]
            S2_M5["2.2_get_CM_similarities_across_modalities_heatmap_venn_upset.ipynb"]
            S2_M1 --> S2_M2 --> S2_M3
            S2_M3 --> S2_M4
            S2_M3 --> S2_M5
        end
        S2_AGG[("Comparison stats")]
        S2_O4 --> S2_AGG
        S2_O5 --> S2_AGG
        S2_M4 --> S2_AGG
        S2_M5 --> S2_AGG
    end

    S1_AGG --> S2_O1
    S1_AGG --> S2_M1

    %% ===== STAGE 3: Compute aCMs =====
    subgraph S3["3. Compute aCMs"]
        direction TB
        S3_C["3.1_get_aCM_scores_clomics.sh"]
        S3_P["3.1_get_aCM_scores_phm.sh"]
        S3_V["3.1_get_aCM_scores_vcmtools.sh"]
        S3_AGG1[("Raw aCM scores<br/>(all methods)")]
        S3_SIGN["3.2_correct_signs_aCM_scores.ipynb"]
        S3_AGG2[("Corrected aCM + cmQTLs")]
        S3_C --> S3_AGG1
        S3_P --> S3_AGG1
        S3_V --> S3_AGG1
        S3_AGG1 --> S3_SIGN --> S3_AGG2
    end

    S1_AGG --> S3_C
    S1_AGG --> S3_P
    S1_AGG --> S3_V

    %% ===== STAGE 4: Visualize =====
    subgraph S4["4. Visualize CMs & ..."]
        direction TB
        subgraph S4_LOOP["Per-ROI Execution"]
            S4_ROI{{"ROI<br/>(region of interest)"}}
            S4_0["4.0_calculate_peak_corr.sh"]
            S4_ROI --> S4_0
        end
        S4_CORR_AGG[("Peak corr matrices<br/>(per ROI × modality/method)")]
        S4_0 --> S4_CORR_AGG
        subgraph S4_VIZ["Visualization Scripts"]
            S4_1["4.1_visualize_peak_based_correlations_for_target_regions.ipynb"]
            subgraph S4_TRACKS["Track Viz (parallel)"]
                S4_2M["4.2_visualize_CM_tracks_across_methods.ipynb"]
                S4_2D["4.2_visualize_CM_tracks_across_modalities.ipynb"]
            end
        end
        S4_CORR_AGG --> S4_1
        S4_CORR_AGG --> S4_2M
        S4_CORR_AGG --> S4_2D
        S4_PLOTS[("Visualizations")]
        S4_1 --> S4_PLOTS
        S4_2M --> S4_PLOTS
        S4_2D --> S4_PLOTS
    end

    S0_AGG --> S4_ROI
    S1_AGG --> S4_0

    %% ===== STAGE 5: cmQTL Analysis =====
    subgraph S5["5. aCM & cmQTL Analysis"]
        direction TB
        S5_1["5.1_compare_QTLs_per_method.ipynb"]
        S5_2["5.2_cmQTLs_analysis.ipynb"]
        S5_OUT[("cmQTL Results")]
        S5_1 --> S5_2 --> S5_OUT
    end

    S3_AGG2 --> S5_1
    S4_PLOTS -.-> S5_2

    %% ===== Final Outputs =====
    S2_AGG --> OUT1[["CM Comparison<br/>Stats & Plots"]]
    S5_OUT --> OUT2[["cmQTL Annotations"]]
    S4_PLOTS --> OUT3[["Track & Corr<br/>Visualizations"]]

    %% Dotted auxiliary edges
    linkStyle 38 stroke-dasharray: 5 5

Loading