Reduce memory allocation when writing results

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Fix writing of net revenue to include all sources of revenue, not just energy revenue (#855).
+- Reduce output-writing memory spikes by caching extracted model values, reusing large scratch arrays, and avoiding copy-heavy transposed DataFrame writes (#908).
 
 ## [0.4.6] - 2026-01-06
 

src/time_domain_reduction/full_time_series_reconstruction.jl

@@ -38,34 +38,39 @@ function full_time_series_reconstruction(
 
     # Get a matrix of the input DataFrame
     DFMatrix = Matrix(DF)
-    # Initialize an array to add the reconstructed data to
-    recon = ["t$t" for t in 1:(TimestepsPerRepPeriod * numPeriods)]
+    total_timesteps = TimestepsPerRepPeriod * numPeriods
 
     # Find the index of the row with the first time step
     t1 = findfirst(x -> x == "t1", DF[!, 1])
 
     # Reconstruction of all hours of the year from TDR
-    for j in range(2, ncol(DF))
-        col = DF[t1:end, j]
-        recon_col = []
-        for i in range(1, numPeriods)
+    recon = Matrix{Any}(undef, total_timesteps, ncol(DF))
+    recon[:, 1] = ["t$t" for t in 1:total_timesteps]
+    for j in 2:ncol(DF)
+        col = DFMatrix[t1:end, j]
+        next_row = 1
+        for i in 1:numPeriods
             index = Period_map[i, "Rep_Period_Index"]
-            recon_temp = col[(TimestepsPerRepPeriod * index - (TimestepsPerRepPeriod - 1)):(TimestepsPerRepPeriod * index)]
-            recon_col = [recon_col; recon_temp]
+            source_start = TimestepsPerRepPeriod * (index - 1) + 1
+            source_end = TimestepsPerRepPeriod * index
+            target_end = next_row + TimestepsPerRepPeriod - 1
+            recon[next_row:target_end, j] = col[source_start:source_end]
+            next_row = target_end + 1
         end
-        recon = [recon recon_col]
     end
-    reconDF = DataFrame(recon, :auto)
+    reconDF = DataFrame(recon, names(DF))
 
     # Insert rows that were above "t1" in the original DataFrame (e.g. "Zone" and "AnnualSum") if present
-    for i in range(1, t1 - 1)
-        insert!(reconDF, i, DFMatrix[i, 1:end], promote = true)
+    if t1 > 1
+        reconDF = vcat(DataFrame(DFMatrix[1:(t1 - 1), :], names(DF)), reconDF)
     end
 
     # Repeat the last rows of the year to fill in the gap (should be 24 hours for non-leap year)
     end_diff = WeightTotal - nrow(reconDF) + 1
-    new_rows = reconDF[(nrow(reconDF) - end_diff):nrow(reconDF), 1:end]
-    new_rows[!, 1] = ["t$t" for t in (WeightTotal - end_diff):WeightTotal]
-    reconDF = [reconDF; new_rows]
+    if end_diff > 0
+        new_rows = copy(reconDF[(nrow(reconDF) - end_diff):nrow(reconDF), 1:end])
+        new_rows[!, 1] = ["t$t" for t in (WeightTotal - end_diff):WeightTotal]
+        append!(reconDF, new_rows)
+    end
     return reconDF
 end

src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl

@@ -22,8 +22,8 @@ function write_reserve_margin_slack(path::AbstractString,
         end
         dfResMar_slack = hcat(dfResMar_slack,
             DataFrame(temp_ResMar_slack, [Symbol("t$t") for t in 1:T]))
-        CSV.write(joinpath(path, "ReserveMargin_slack_and_penalties.csv"),
-            dftranspose(dfResMar_slack, false),
+        write_transposed_csv(joinpath(path, "ReserveMargin_slack_and_penalties.csv"),
+            dfResMar_slack,
             writeheader = false)
     end
     return nothing

src/write_outputs/dftranspose.jl

@@ -1,26 +1,37 @@
 ################################################################################
-## function dftranspose(df)
-##
-## inputs: df - [DataFrame] a DataFrame object to be transposed
-## results t - [DataFrame] a transposed version of the df DataFrame.
-##		   withhead - [Boolean] if True, first column of df will become column
-##		   names for t. Otherwise, first column will first row and column names
-##		   will be generic (e.g. x1:xN)
-##
-## Note this function is necessary because no stock function to transpose
-## DataFrames appears to exist.
+## helpers for writing transposed output tables
 ################################################################################
 @doc raw"""
-   df = dftranspose(df::DataFrame, withhead::Bool)
+    transpose_output_dataframe(df::DataFrame; withhead::Bool=false)
 
-Returns a transpose of a Dataframe.
+Return a transposed copy of an output DataFrame without collecting each row into
+an intermediate vector.
 """
-function dftranspose(df::DataFrame, withhead::Bool)
-    if withhead
-        colnames = cat(:Row, Symbol.(df[!, 1]), dims = 1)
-        return DataFrame([[names(df)]; collect.(eachrow(df))], colnames)
-    else
-        return DataFrame([[names(df)]; collect.(eachrow(df))],
-            [:Row; Symbol.("x", axes(df, 1))])
+function transpose_output_dataframe(df::DataFrame; withhead::Bool = false)
+    row_count, col_count = size(df)
+    colnames = withhead ? Symbol[:Row; Symbol.(df[!, 1])] :
+               Symbol[:Row; Symbol.("x", 1:row_count)]
+    transposed = Matrix{Any}(undef, col_count, row_count + 1)
+    headers = names(df)
+
+    for col in 1:col_count
+        transposed[col, 1] = headers[col]
+    end
+
+    for row in 1:row_count
+        for col in 1:col_count
+            transposed[col, row + 1] = df[row, col]
+        end
+    end
+
+    return DataFrame(transposed, colnames)
+end
+
+function write_transposed_csv(filename::AbstractString, df::DataFrame; kwargs...)
+    options = Dict{Symbol, Any}(pairs(kwargs))
+    if haskey(options, :writeheader) && !haskey(options, :header)
+        options[:header] = options[:writeheader]
+        delete!(options, :writeheader)
     end
-end # End dftranpose()
+    return CSV.write(filename, transpose_output_dataframe(df); options...)
+end

src/write_outputs/hourly_matching_requirement/write_hourly_matching_prices.jl

@@ -23,7 +23,8 @@ function write_hourly_matching_prices(path::AbstractString,
     auxNew_Names = [Symbol("Zone"); [Symbol("t$t") for t in 1:T]]
     rename!(dfHourlyMatchPrices, auxNew_Names)
 
-    CSV.write(joinpath(path, "hourly_matching_prices.csv"),
-        dftranspose(dfHourlyMatchPrices, false), writeheader = false)
+    write_transposed_csv(joinpath(path, "hourly_matching_prices.csv"),
+        dfHourlyMatchPrices,
+        writeheader = false)
     return nothing
 end

src/write_outputs/hourly_matching_requirement/write_hourly_matching_slack.jl

@@ -28,8 +28,8 @@ function write_hourly_matching_slack(path::AbstractString,
         end
         dfHM_slack = hcat(dfHM_slack,
             DataFrame(temp_HM_slack, [Symbol("t$t") for t in 1:T]))
-        CSV.write(joinpath(path, "HourlyMatching_slack_and_penalties.csv"),
-            dftranspose(dfHM_slack, false),
+        write_transposed_csv(joinpath(path, "HourlyMatching_slack_and_penalties.csv"),
+            dfHM_slack,
             writeheader = false)
     end
     return nothing

src/write_outputs/long_duration_storage/write_opwrap_lds_dstor.jl

@@ -26,7 +26,5 @@ function write_opwrap_lds_dstor(path::AbstractString, inputs::Dict, setup::Dict,
     dfdStorage = hcat(dfdStorage, DataFrame(dsoc, :auto))
     auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("w$t") for t in 1:W]]
     rename!(dfdStorage, auxNew_Names)
-    CSV.write(joinpath(path, "dStorage.csv"),
-        dftranspose(dfdStorage, false),
-        header = false)
+    write_transposed_csv(joinpath(path, "dStorage.csv"), dfdStorage, header = false)
 end

src/write_outputs/long_duration_storage/write_opwrap_lds_stor_init.jl

@@ -29,8 +29,8 @@ function write_opwrap_lds_stor_init(path::AbstractString,
     dfStorageInit = hcat(dfStorageInit, DataFrame(socw, :auto))
     auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("n$t") for t in 1:NPeriods]]
     rename!(dfStorageInit, auxNew_Names)
-    CSV.write(joinpath(path, "StorageInit.csv"),
-        dftranspose(dfStorageInit, false),
+    write_transposed_csv(joinpath(path, "StorageInit.csv"),
+        dfStorageInit,
         header = false)
 
     # Write storage evolution over full time horizon
@@ -80,6 +80,6 @@ function write_opwrap_lds_stor_init(path::AbstractString,
     df_SOC_t = hcat(df_SOC_t, DataFrame(SOC_t, :auto))
     auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("n$t") for t in 1:T_hor]]
     rename!(df_SOC_t,auxNew_Names)
-    CSV.write(joinpath(path, "StorageEvol.csv"), dftranspose(df_SOC_t, false), writeheader=false)
+    write_transposed_csv(joinpath(path, "StorageEvol.csv"), df_SOC_t, writeheader = false)
 
 end

src/write_outputs/output_cache.jl

@@ -0,0 +1,181 @@
+@doc raw"""
+        OutputCache
+
+Container for extracted model outputs reused across write-output functions to
+reduce repeated JuMP value materialization and large temporary allocations.
+
+# Fields
+- `scale_factor::Float64`: Output scaling multiplier (1 or `ModelScalingFactor`).
+- `resource_time_scratch::Matrix{Float64}`: Reusable `G x T` workspace.
+- `price::Union{Nothing, Matrix{Float64}}`: Cached locational marginal prices (`T x Z`).
+- `vP::Matrix{Float64}`: Cached dispatch (`G x T`).
+- `eTotalCap::Vector{Float64}`: Cached endogenously available capacity (`G`).
+- `vCHARGE`, `vCHARGE_FLEX`, `vUSE`, `vCHARGE_VRE_STOR`, `vCHARGE_ALLAM`:
+    optional cached charging/consumption matrices.
+- `vS`, `vS_HYDRO`, `vS_FLEX`, `vS_VRE_STOR`: optional cached storage-state matrices.
+- `vNSE::Union{Nothing, Array{Float64,3}}`: optional cached non-served energy tensor.
+- `eEmissionsByZone::Union{Nothing, Matrix{Float64}}`: optional cached zonal emissions.
+"""
+struct OutputCache
+    scale_factor::Float64
+    resource_time_scratch::Matrix{Float64}
+    price::Union{Nothing, Matrix{Float64}}
+    vP::Matrix{Float64}
+    eTotalCap::Vector{Float64}
+    vCHARGE::Union{Nothing, Matrix{Float64}}
+    vCHARGE_FLEX::Union{Nothing, Matrix{Float64}}
+    vUSE::Union{Nothing, Matrix{Float64}}
+    vCHARGE_VRE_STOR::Union{Nothing, Matrix{Float64}}
+    vCHARGE_ALLAM::Union{Nothing, Matrix{Float64}}
+    vS::Union{Nothing, Matrix{Float64}}
+    vS_HYDRO::Union{Nothing, Matrix{Float64}}
+    vS_FLEX::Union{Nothing, Matrix{Float64}}
+    vS_VRE_STOR::Union{Nothing, Matrix{Float64}}
+    vNSE::Union{Nothing, Array{Float64, 3}}
+    eEmissionsByZone::Union{Nothing, Matrix{Float64}}
+end
+
+_extract_output_data(var::JuMP.Containers.DenseAxisArray) = var.data
+_extract_output_data(var::AbstractArray) = var
+
+_extract_output_matrix(var)::Matrix{Float64} = Matrix{Float64}(Array(value.(
+    _extract_output_data(var))))
+
+_extract_output_vector(var)::Vector{Float64} = vec(Array(value.(
+    _extract_output_data(var))))
+
+@doc raw"""
+    build_output_cache(
+        EP::Model,
+        inputs::Dict,
+        setup::Dict,
+        output_settings_d::Dict = setup["WriteOutputsSettingsDict"];
+        selective::Bool = false)
+
+Build an `OutputCache` for write-output routines.
+
+When `selective=true`, only values needed by enabled outputs in
+`output_settings_d` are materialized; otherwise, common output matrices are
+eagerly extracted once.
+
+# Arguments
+- `EP::Model`: Solved JuMP model.
+- `inputs::Dict`: Parsed GenX input data.
+- `setup::Dict`: Run settings.
+- `output_settings_d::Dict`: Output toggles.
+- `selective::Bool=false`: Enable selective extraction.
+
+# Returns
+- `OutputCache`: Cache object shared by write-output functions.
+"""
+function build_output_cache(
+    EP::Model,
+    inputs::Dict,
+    setup::Dict,
+    output_settings_d::Dict = setup["WriteOutputsSettingsDict"];
+    selective::Bool = false)
+    scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1.0
+    G = inputs["G"]
+    T = inputs["T"]
+
+    needs_power = output_settings_d["WritePower"] ||
+                  output_settings_d["WriteCapacityFactor"] ||
+                  output_settings_d["WriteCurtailment"] ||
+                  output_settings_d["WriteEnergyRevenue"] ||
+                  output_settings_d["WriteChargingCost"] ||
+                  output_settings_d["WriteNetRevenue"]
+    needs_capacity = output_settings_d["WriteCapacityFactor"] ||
+                     output_settings_d["WriteCurtailment"]
+    needs_charge = output_settings_d["WriteCharge"] ||
+                   output_settings_d["WriteChargingCost"] ||
+                   output_settings_d["WriteNetRevenue"]
+    needs_storage = output_settings_d["WriteStorage"] ||
+                    output_settings_d["WriteStorageDual"] ||
+                    output_settings_d["WriteNetRevenue"]
+    needs_price = has_duals(EP) == 1 && (
+        output_settings_d["WritePrice"] ||
+        output_settings_d["WriteEnergyRevenue"] ||
+        output_settings_d["WriteChargingCost"] ||
+        output_settings_d["WriteNetRevenue"])
+    needs_scratch = needs_power || needs_capacity || needs_storage ||
+                    output_settings_d["WriteNSE"] ||
+                    output_settings_d["WriteEmissions"] ||
+                    output_settings_d["WriteEnergyRevenue"] ||
+                    output_settings_d["WriteChargingCost"]
+
+    return OutputCache(
+        scale_factor,
+        selective && !needs_scratch ? zeros(Float64, 0, 0) : zeros(Float64, G, T),
+        needs_price ? locational_marginal_price(EP, inputs, setup) : nothing,
+        selective && !needs_power ? zeros(Float64, 0, 0) : _extract_output_matrix(EP[:vP]),
+        selective && !needs_capacity ? zeros(Float64, 0) : _extract_output_vector(EP[:eTotalCap]),
+        selective && !needs_charge || isempty(inputs["STOR_ALL"]) ? nothing : _extract_output_matrix(EP[:vCHARGE]),
+        selective && !needs_charge || isempty(inputs["FLEX"]) ? nothing : _extract_output_matrix(EP[:vCHARGE_FLEX]),
+        selective && !needs_charge || isempty(inputs["ELECTROLYZER"]) ? nothing : _extract_output_matrix(EP[:vUSE]),
+        selective && !needs_charge || isempty(inputs["VRE_STOR"]) ? nothing : _extract_output_matrix(EP[:vCHARGE_VRE_STOR]),
+        selective && !needs_charge || isempty(inputs["ALLAM_CYCLE_LOX"]) ? nothing : _extract_output_matrix(EP[:vCHARGE_ALLAM]),
+        selective && !needs_storage || isempty(inputs["STOR_ALL"]) ? nothing : _extract_output_matrix(EP[:vS]),
+        selective && !needs_storage || isempty(inputs["HYDRO_RES"]) ? nothing : _extract_output_matrix(EP[:vS_HYDRO]),
+        selective && !needs_storage || isempty(inputs["FLEX"]) ? nothing : _extract_output_matrix(EP[:vS_FLEX]),
+        selective && !needs_storage || isempty(inputs["VRE_STOR"]) ? nothing : _extract_output_matrix(EP[:vS_VRE_STOR]),
+        output_settings_d["WriteNSE"] ? Array{Float64, 3}(Array(value.(EP[:vNSE]))) : nothing,
+        output_settings_d["WriteEmissions"] ? _extract_output_matrix(EP[:eEmissionsByZone]) : nothing)
+end
+
+@doc raw"""
+    resource_time_scratch!(cache::OutputCache) -> Matrix{Float64}
+
+Clear and return the reusable `G x T` scratch matrix stored in `cache`.
+"""
+function resource_time_scratch!(cache::OutputCache)::Matrix{Float64}
+    fill!(cache.resource_time_scratch, 0.0)
+    return cache.resource_time_scratch
+end
+
+@doc raw"""
+    scaled_resource_time_matrix!(cache::OutputCache, data::Matrix{Float64})
+
+Return `data` scaled by `cache.scale_factor` while avoiding unnecessary
+allocations. If scaling is not needed, returns `data`; otherwise, writes the
+scaled result into cache scratch memory and returns that scratch matrix.
+"""
+function scaled_resource_time_matrix!(cache::OutputCache, data::Matrix{Float64})
+    if cache.scale_factor == 1
+        return data
+    end
+    scratch = resource_time_scratch!(cache)
+    copyto!(scratch, data)
+    rmul!(scratch, cache.scale_factor)
+    return scratch
+end
+
+@doc raw"""
+    materialize_output_blocks(blocks, block_ids::Vector{Vector{Int}}, T::Int)
+
+Stack a list of `blocks` (each with `T` columns) into one dense matrix and the
+corresponding flattened resource-id vector.
+
+# Arguments
+- `blocks`: Row blocks to stack.
+- `block_ids::Vector{Vector{Int}}`: Resource ids for each block row.
+- `T::Int`: Number of time steps (columns).
+
+# Returns
+- `(data, ids)`: `data::Matrix{Float64}` and `ids::Vector{Int}`.
+"""
+function materialize_output_blocks(blocks, block_ids::Vector{Vector{Int}}, T::Int)
+    total_rows = sum(length, block_ids; init = 0)
+    data = Matrix{Float64}(undef, total_rows, T)
+    ids = Vector{Int}(undef, total_rows)
+
+    next_row = 1
+    for (block, ids_block) in zip(blocks, block_ids)
+        row_count = length(ids_block)
+        row_range = next_row:(next_row + row_count - 1)
+        data[row_range, :] .= block
+        ids[row_range] .= ids_block
+        next_row += row_count
+    end
+
+    return data, ids
+end

src/write_outputs/reserves/write_rsv.jl

@@ -30,8 +30,8 @@ function write_rsv(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
         rename!(total, auxNew_Names)
         rename!(unmet, auxNew_Names)
         dfRsv = vcat(dfRsv, unmet, total)
-        CSV.write(joinpath(path, "reserves.csv"),
-            dftranspose(dfRsv, false),
+        write_transposed_csv(joinpath(path, "reserves.csv"),
+            dfRsv,
             writeheader = false)
     end
 end

src/write_outputs/transmission/write_transmission_flows.jl

@@ -22,7 +22,7 @@ function write_transmission_flows(path::AbstractString,
         dfFlow = hcat(dfFlow, DataFrame(flow, :auto))
         auxNew_Names = [Symbol("Line"); [Symbol("t$t") for t in 1:T]]
         rename!(dfFlow, auxNew_Names)
-        CSV.write(filepath, dftranspose(dfFlow, false), writeheader = false)
+        write_transposed_csv(filepath, dfFlow, writeheader = false)
 
         if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
             write_full_time_series_reconstruction(path, setup, dfFlow, "flow")

src/write_outputs/transmission/write_transmission_losses.jl

@@ -27,8 +27,8 @@ function write_transmission_losses(path::AbstractString,
             auxNew_Names)
         total[:, 3:(T + 2)] .= sum(tlosses, dims = 1)
         dfTLosses = vcat(dfTLosses, total)
-        CSV.write(joinpath(path, "tlosses.csv"),
-            dftranspose(dfTLosses, false),
+        write_transposed_csv(joinpath(path, "tlosses.csv"),
+            dfTLosses,
             writeheader = false)
 
         if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1