Use zips for projects

Author: tommitytom

CMakeLists.txt

@@ -225,7 +225,8 @@ add_executable(rpc-schema-dump
     src/system/mesen/GbaSystem.cpp
     src/system/mesen/roles/NesN8MidiRole.cpp)
 target_include_directories(rpc-schema-dump PRIVATE src)
-target_link_libraries(rpc-schema-dump PRIVATE rpcpp sameboy mesen r8brain enkiTS miniaudio efsw)
+target_include_directories(rpc-schema-dump PRIVATE deps/lv_binding_js/deps/txiki/deps/miniz)
+target_link_libraries(rpc-schema-dump PRIVATE rpcpp sameboy mesen r8brain enkiTS miniaudio efsw miniz)
 
 # --- UI bundle: esbuild-bundled TS/TSX, embedded into the plugin as a byte array ---
 find_program(NODE_EXECUTABLE node REQUIRED)
@@ -379,6 +380,11 @@ target_link_libraries(${NAME} PUBLIC miniaudio)
 # PluginRpcService.cpp (shared between FILES_DSP and FILES_UI) references
 # UserConfig symbols and we don't split the .cpp.
 target_link_libraries(${NAME} PUBLIC efsw)
+# Project files are PKZIP blobs (project.json + raw binary entries); miniz
+# provides the in-memory writer/reader. The target is built as part of txiki
+# but we link + include directly so src/util/MinizZip.hpp can #include "miniz.h".
+target_link_libraries(${NAME} PUBLIC miniz)
+target_include_directories(${NAME} PUBLIC deps/lv_binding_js/deps/txiki/deps/miniz)
 
 # Force UI artifacts to refresh before any consumer of bundle_data.c
 # builds. Wiring only ${NAME}-ui (DPF's static-lib aggregator) is not

src/PluginDSP.cpp

@@ -17,6 +17,7 @@
 #include "lsdj/SampleCache.hpp"
 #include "project/ProjectSerialization.hpp"
 #include "system/SystemTypes.hpp"
+#include "util/Base64.hpp"
 #include "system/sameboy/SameBoyConfig.hpp"
 #include "system/sameboy/SameBoySystem.hpp"
 #include "system/sameboy/roles/LsdjKitPatchRole.hpp"
@@ -129,8 +130,13 @@ class LVGLPluginDSP : public Plugin {
     {
         if (std::strcmp(key, "project") != 0) return String();
         try {
-            const std::string json = projectConfigToJson(project.snapshotConfig());
-            return String(json.c_str());
+            const auto zip = projectConfigToZip(project.snapshotConfig());
+            // DPF state is a NUL-terminated UTF-8 string, so the binary zip
+            // blob is wrapped in base64. The inner zip is already deflated,
+            // so the base64 layer's 33% overhead applies to compressed bytes
+            // — still a large net win over today's JSON+inline-base64.
+            const std::string encoded = base64::encode(zip);
+            return String(encoded.c_str());
         } catch (const std::exception& e) {
             d_stderr("[PluginDSP] getState serialization failed: %s", e.what());
             return String();
@@ -140,42 +146,36 @@ class LVGLPluginDSP : public Plugin {
     void setState(const char* key, const char* value) override
     {
         if (std::strcmp(key, "project") != 0) return;
-        applyProjectFromJson(value);
-    }
-
-    // Replace the running project with one parsed from a JSON blob. Shared by
-    // DPF setState (host-driven save/restore) and Command::LoadProject (the
-    // user-driven "Load project" menu entry, fed in via the UI thread).
-    // Caller is responsible for the threading context — setState runs DSP-side
-    // before activate; Command::LoadProject runs DSP-side during the run-loop
-    // command drain, same window AddSystem/etc. already mutate the project.
-    void applyProjectFromJson(const char* json)
-    {
-        if (json == nullptr || json[0] == '\0') return;
-
-        std::optional<ProjectConfig> parsed;
-        try {
-            parsed = projectConfigFromJson(std::string_view(json));
-        } catch (const std::exception& e) {
-            d_stderr("[PluginDSP] applyProjectFromJson parse exception: %s", e.what());
-            return;
-        }
+        if (value == nullptr || value[0] == '\0') return;
+        const auto decoded = base64::decode(value);
+        auto parsed = projectConfigFromZip(decoded);
         if (!parsed) {
-            d_stderr("[PluginDSP] applyProjectFromJson: failed to parse project JSON");
+            d_stderr("[PluginDSP] setState: failed to parse project zip");
             return;
         }
+        applyProjectFromConfig(*parsed);
+    }
 
+    // Replace the running project with a fully-parsed config. Shared by DPF
+    // setState (host-driven save/restore) and Command::LoadProject (the
+    // user-driven "Load project" menu entry, fed in via the UI thread which
+    // does the file IO and zip parse). Caller is responsible for the
+    // threading context — setState runs DSP-side before activate;
+    // Command::LoadProject runs DSP-side during the run-loop command drain,
+    // same window AddSystem/etc. already mutate the project.
+    void applyProjectFromConfig(const ProjectConfig& parsed)
+    {
         // Tear down current systems. Non-RT (deletes GB instances) — same
         // category of work AddSystem/RemoveSystem already do during command
         // drain.
         project.clearSystems();
         project.config() = ProjectConfig{};
 
         SystemId firstAdded = 0;
-        for (const auto& sysConfig : parsed->systems) {
+        for (const auto& sysConfig : parsed.systems) {
             const SystemId id = project.addSystem(sysConfig);
             if (id == 0) {
-                d_stderr("[PluginDSP] applyProjectFromJson: addSystem failed for one entry");
+                d_stderr("[PluginDSP] applyProjectFromConfig: addSystem failed for one entry");
                 continue;
             }
             if (firstAdded == 0) firstAdded = id;
@@ -184,7 +184,7 @@ class LVGLPluginDSP : public Plugin {
 
         // Notify the UI (if attached) so it drops its cached project view.
         if (!events.tryPush(Event::makeConfigChanged()))
-            d_stderr("[PluginDSP] applyProjectFromJson: event queue full; dropping ConfigChanged");
+            d_stderr("[PluginDSP] applyProjectFromConfig: event queue full; dropping ConfigChanged");
     }
 
     // ----------------------------------------------------------------------------------------------------------------
@@ -445,7 +445,7 @@ class LVGLPluginDSP : public Plugin {
                             kitCfg->kits.push_back(std::move(fresh));
                             slot = &kitCfg->kits.back();
                         }
-                        slot->compiledBytes = Base64Bytes(*owned);
+                        slot->compiledBytes = *owned;
                         slot->compiledHash  =
                             rp::lsdj::SampleCache::hashBytes(owned->data(), owned->size());
                         break;
@@ -466,19 +466,19 @@ class LVGLPluginDSP : public Plugin {
                 } break;
 
                 case Command::Kind::LoadProject: {
-                    std::string* json = cmd.payload.loadProject.json;
-                    if (json) {
-                        applyProjectFromJson(json->c_str());
-                        // applyProjectFromJson constructs SameBoySystems via
+                    ProjectConfig* config = cmd.payload.loadProject.config;
+                    if (config) {
+                        applyProjectFromConfig(*config);
+                        // applyProjectFromConfig constructs SameBoySystems via
                         // Project::addSystem but doesn't activate them — the
                         // setState path relies on DPF calling activate() after
                         // setState, but Command::LoadProject runs mid-run()
                         // with no following activate. Activate now so the new
                         // systems actually start emulating; SameBoySystem's
                         // onActivate is idempotent for already-active systems.
                         project.onActivate(fSampleRate);
-                        delete json;
-                        // applyProjectFromJson already pushes ConfigChanged.
+                        delete config;
+                        // applyProjectFromConfig already pushes ConfigChanged.
                         // Don't double-emit by setting projectMutated.
                     }
                 } break;

src/PluginRpcService.cpp

@@ -7,6 +7,7 @@
 #include <filesystem>
 #include <fstream>
 #include <memory>
+#include <span>
 #include <string_view>
 #include <utility>
 #include <vector>
@@ -68,6 +69,14 @@ bool spillString(const std::string& path, const std::string& data) {
     return out.good();
 }
 
+bool spillBytes(const std::string& path, std::span<const std::uint8_t> data) {
+    std::ofstream out(path, std::ios::binary | std::ios::trunc);
+    if (!out) return false;
+    out.write(reinterpret_cast<const char*>(data.data()),
+              static_cast<std::streamsize>(data.size()));
+    return out.good();
+}
+
 } // namespace
 
 PluginRpcService::PluginRpcService(Project* project,
@@ -149,7 +158,7 @@ SystemBase* PluginRpcService::buildSystemFromPath(const std::string& path) {
         sav.replace_extension(".sav");
         std::vector<std::uint8_t> sramBytes = slurp(sav.string());
         if (!sramBytes.empty())
-            cfg.sram = Base64Bytes(std::move(sramBytes));
+            cfg.sram = std::move(sramBytes);
     }
 
     auto sys = std::make_unique<SameBoySystem>(id, cfg, std::move(bytes));
@@ -219,15 +228,20 @@ bool PluginRpcService::saveProjectToPath(const std::string& path) {
         emit("project-error", path);
         return false;
     }
-    std::string json;
+    std::vector<std::uint8_t> zip;
     try {
-        json = projectConfigToJson(project_->snapshotConfig());
+        zip = projectConfigToZip(project_->snapshotConfig());
     } catch (const std::exception& e) {
         std::fprintf(stderr, "saveProjectToPath: serialize failed: %s\n", e.what());
         emit("project-error", path);
         return false;
     }
-    if (!spillString(path, json)) {
+    if (zip.empty()) {
+        std::fprintf(stderr, "saveProjectToPath: zip serialization produced empty buffer\n");
+        emit("project-error", path);
+        return false;
+    }
+    if (!spillBytes(path, zip)) {
         std::fprintf(stderr, "saveProjectToPath: write failed for '%s'\n", path.c_str());
         emit("project-error", path);
         return false;
@@ -243,14 +257,20 @@ bool PluginRpcService::loadProjectFromPath(const std::string& path) {
         emit("project-error", path);
         return false;
     }
-    std::string json = slurpString(path);
-    if (json.empty()) {
+    const auto bytes = slurp(path);
+    if (bytes.empty()) {
         std::fprintf(stderr, "loadProjectFromPath: empty / unreadable '%s'\n", path.c_str());
         emit("project-error", path);
         return false;
     }
-    // Heap-allocate the JSON; DSP frees after parsing.
-    auto* heap = new std::string(std::move(json));
+    auto parsed = projectConfigFromZip(bytes);
+    if (!parsed) {
+        std::fprintf(stderr, "loadProjectFromPath: failed to parse zip '%s'\n", path.c_str());
+        emit("project-error", path);
+        return false;
+    }
+    // Heap-allocate the parsed config; DSP frees after applying.
+    auto* heap = new ProjectConfig(std::move(*parsed));
     if (!commands_->tryPush(Command::makeLoadProject(heap))) {
         std::fprintf(stderr, "loadProjectFromPath: command queue full\n");
         delete heap;

src/project/Project.cpp

@@ -44,7 +44,7 @@ SystemId Project::addSystem(const SystemConfig& config) {
         // Prefer embedded ROM bytes (round-tripped through DPF state) over
         // re-reading from disk. Only fall back to slurpFile when bytes are
         // absent — covers legacy/dev paths and the embedRom=false opt-out.
-        std::vector<std::uint8_t> rom = sb->romBytes.bytes();
+        std::vector<std::uint8_t> rom = sb->romBytes;
         if (rom.empty())
             rom = slurpFile(sb->romPath);
         if (rom.empty()) {
@@ -60,7 +60,7 @@ SystemId Project::addSystem(const SystemConfig& config) {
     }
 
     if (const auto* mb = rfl::get_if<MesenConfig>(&config.variant())) {
-        std::vector<std::uint8_t> rom = mb->romBytes.bytes();
+        std::vector<std::uint8_t> rom = mb->romBytes;
         if (rom.empty())
             rom = slurpFile(mb->romPath);
         if (rom.empty()) {
@@ -78,7 +78,7 @@ SystemId Project::addSystem(const SystemConfig& config) {
     }
 
     if (const auto* gb = rfl::get_if<GbaSystemConfig>(&config.variant())) {
-        std::vector<std::uint8_t> rom = gb->romBytes.bytes();
+        std::vector<std::uint8_t> rom = gb->romBytes;
         if (rom.empty())
             rom = slurpFile(gb->romPath);
         if (rom.empty()) {