Feature/compression plugin

conan/api/conan_api.py

@@ -135,6 +135,25 @@ def __init__(self, conan_api):
             self.cache = PkgCache(self._conan_api.home_folder, self.global_conf)
             self._settings_yml = None
             self._remote_manager = None
+            self._compression_plugin = None
+
+        @property
+        def compression_plugin(self):
+            # NOTE: Users cannot store a reference to this plugin, otherwise it will not
+            # be updated after a "conan config install/install-pkg"
+            if self._compression_plugin is None:
+                compression_plugin_path = HomePaths(
+                    self._conan_api.home_folder).compression_plugin_path
+                if not os.path.exists(compression_plugin_path):
+                    self._compression_plugin = False  # Avoid FS re-check
+                    return None
+                mod, _ = load_python_file(compression_plugin_path)
+                # A plugin can provide just 1 of them
+                if not hasattr(mod, "tar_extract") and not hasattr(mod, "tar_compress"):
+                    raise ConanException("The 'compression.py' plugin does not contain "
+                                         "required `tar_extract` or `tar_compress` functions")
+                self._compression_plugin = mod
+            return self._compression_plugin
 
         def set_core_confs(self, core_confs):
             confs = ConfDefinition()
@@ -165,6 +184,7 @@ def reinit(self):
             self.cache = PkgCache(self._conan_api.home_folder, self.global_conf)
             self._remote_manager = None
             self._editable_packages = EditablePackages(self._conan_api.home_folder)
+            self._compression_plugin = None
 
         @property
         def settings_yml(self):
@@ -180,7 +200,8 @@ def remote_manager(self):
                 requester = self._conan_api._api_helpers.requester  # noqa
                 auth_manager = ConanApiAuthManager(requester, self._conan_api.home_folder, localdb,
                                                    self.global_conf)
-                self._remote_manager = RemoteManager(self.cache, auth_manager, home_folder)
+                self._remote_manager = RemoteManager(self.cache, auth_manager, home_folder,
+                                                     self.compression_plugin)
             return self._remote_manager
 
         @property

conan/api/subapi/cache.py

@@ -194,9 +194,10 @@ def sign(self, package_list):
                 "https://docs.conan.io/2/reference/extensions/package_signing.html.")
 
         loader = self._api_helpers.loader
+        compress_plugin = self._api_helpers.compression_plugin
         preparator = PackagePreparator(loader, self._api_helpers.cache,
                                        self._api_helpers.remote_manager,
-                                       self._api_helpers.global_conf)
+                                       self._api_helpers.global_conf, compress_plugin)
         # Some packages can have missing sources/exports_sources
         enabled_remotes = self._conan_api.remotes.list()
         preparator.prepare(package_list, enabled_remotes, None, force=True)
@@ -373,7 +374,11 @@ def save(self, package_list: PackagesList, path, no_source=False) -> None:
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, tgz_name, os.path.dirname(path), compresslevel, recursive=True)
+
+        plugin = self._conan_api._api_helpers.compression_plugin  # noqa
+        compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None
+        compress_files(tar_files, tgz_name, os.path.dirname(path), compresslevel,
+                       recursive=True, compress_plugin=compress_plugin)
         remove(pkglist_path)
         ConanOutput().success(f"Created cache save file: {path}")
 
@@ -391,13 +396,26 @@ def restore(self, path) -> PackagesList:
         cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf)
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
 
-        with open(path, mode='rb') as file_handler:
-            the_tar = tarfile.open(fileobj=file_handler)
-            fileobj = the_tar.extractfile("pkglist.json")
-            pkglist = fileobj.read()
-            the_tar.extraction_filter = (lambda member, _: member)  # fully_trusted (Py 3.14)
-            the_tar.extractall(path=cache_folder)
-            the_tar.close()
+        plugin = self._conan_api._api_helpers.compression_plugin  # noqa
+        extract_plugin = getattr(plugin, "tar_extract", None) if plugin else None
+        extracted = False
+        if extract_plugin:
+            extracted = extract_plugin(path, cache_folder, scope="Restore")
+            # If the plugin returns false, fallback to Conan extraction
+
+        if extracted is False:
+            with open(path, mode='rb') as file_handler:
+                the_tar = tarfile.open(fileobj=file_handler)
+                the_tar.extraction_filter = (lambda member, _: member)  # fully_trusted (Py 3.14)
+                the_tar.extractall(path=cache_folder)
+                the_tar.close()
+
+        # Retrieve the package list from the already extracted archive
+        pkglist_path = os.path.join(cache_folder, "pkglist.json")
+        with open(pkglist_path) as file_handler:
+            pkglist = file_handler.read()
+        # Delete the pkglist.json file to keep cache clean
+        remove(pkglist_path)
 
         # After unzipping the files, we need to update the DB that references these files
         out = ConanOutput()

conan/api/subapi/upload.py

@@ -64,9 +64,10 @@ def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote],
             raise ConanException("Empty string and patterns can not be mixed for metadata.")
 
         loader = self._api_helpers.loader
+        compress_plugin = self._api_helpers.compression_plugin
         preparator = PackagePreparator(loader, self._api_helpers.cache,
                                        self._api_helpers.remote_manager,
-                                       self._api_helpers.global_conf)
+                                       self._api_helpers.global_conf, compress_plugin)
         preparator.prepare(package_list, enabled_remotes, metadata)
         signer = PkgSignaturesPlugin(self._api_helpers.cache, self._conan_api.home_folder)
         if signer.is_sign_configured:

conan/internal/api/uploader.py

@@ -103,7 +103,7 @@ def get_compress_level(compressformat, global_conf):
 
 
 class PackagePreparator:
-    def __init__(self, loader, cache, remote_manager, global_conf):
+    def __init__(self, loader, cache, remote_manager, global_conf, compress_plugin):
         self._loader = loader
         self._remote_manager = remote_manager
         self._cache = cache
@@ -113,6 +113,8 @@ def __init__(self, loader, cache, remote_manager, global_conf):
         compresslevel = get_compress_level(compressformat, global_conf)
         self._compressformat = compressformat
         self._compresslevel = compresslevel
+        plugin = compress_plugin  # noqa
+        self._compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None
 
     def prepare(self, pkg_list, enabled_remotes, metadata, force=False):
         local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"])
@@ -243,7 +245,8 @@ def _compressed_file(self, filename, files, download_folder, ref):
         file_name = filename + self._compressformat
         package_file = os.path.join(download_folder, file_name)
         compressed_path = compress_files(files, file_name, download_folder,
-                                         compresslevel=self._compresslevel, scope=str(ref))
+                                         compresslevel=self._compresslevel, scope=str(ref),
+                                         compress_plugin=self._compress_plugin)
         assert compressed_path == package_file
         assert os.path.exists(package_file)
         return file_name
@@ -336,13 +339,22 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False):
+def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False,
+                   compress_plugin=None):
     t1 = time.time()
     tgz_path = os.path.join(dest_dir, name)
 
     out = ConanOutput(scope=scope)
     out.info(f"Compressing {name}")
 
+    if compress_plugin is not None:
+        out.info(f"Compressing {name} using compression plugin")
+        compressed = compress_plugin(archive_path=tgz_path, files=files, recursive=recursive,
+                                     compresslevel=compresslevel, scope=scope)
+        if compressed is not False:
+            out.debug(f"{name} compressed in {time.time() - t1} time")
+            return tgz_path
+
     if name.endswith("zst"):
         with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar:  # noqa Py314 only
             for filename, abs_path in sorted(files.items()):

conan/internal/cache/home_paths.py

@@ -85,3 +85,7 @@ def settings_path_user(self):
     @property
     def config_version_path(self):
         return os.path.join(self._home, "config_version.json")
+
+    @property
+    def compression_plugin_path(self):
+        return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py")

conan/internal/rest/remote_manager.py

@@ -28,11 +28,13 @@ class RemoteManager:
 
     _ErrorMsg = namedtuple("ErrorMsg", ["message"])
 
-    def __init__(self, cache, auth_manager, home_folder):
+    def __init__(self, cache, auth_manager, home_folder, compression_plugin=None):
         self._cache = cache
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache, home_folder)
         self._home_folder = home_folder
+        self._extract_plugin = getattr(compression_plugin, "tar_extract", None) \
+            if compression_plugin else None
 
     def _local_folder_remote(self, remote):
         if remote.remote_type == LOCAL_RECIPES_INDEX:
@@ -99,7 +101,8 @@ def _download_recipe(self, layout, ref, remote, metadata):
         tgz_file = zipped_files.pop(export_file, None)
 
         if tgz_file:
-            uncompress_file(tgz_file, export_folder, scope=str(ref))
+            uncompress_file(tgz_file, export_folder, scope=str(ref),
+                            extract_plugin=self._extract_plugin)
         mkdir(export_folder)
         for file_name, file_path in zipped_files.items():  # copy CONANFILE
             shutil.move(file_path, os.path.join(export_folder, file_name))
@@ -136,7 +139,8 @@ def get_recipe_sources(self, ref, layout, remote):
                 self._signer.verify(ref, download_folder, layout.metadata(), files=zipped_files)
                 # Only 1 file is guaranteed
                 tgz_file = next(iter(zipped_files.values()))
-                uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
+                uncompress_file(tgz_file, export_sources_folder, scope=str(ref),
+                                extract_plugin=self._extract_plugin)
 
     def get_package(self, pref, remote, metadata=None):
         output = ConanOutput(scope=str(pref.ref))
@@ -191,7 +195,8 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
 
             tgz_file = zipped_files.pop(package_file)
             package_folder = layout.package()
-            uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
+            uncompress_file(tgz_file, package_folder, scope=str(pref.ref),
+                            extract_plugin=self._extract_plugin)
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
             for file_name, file_path in zipped_files.items():  # copy CONANINFO and CONANMANIFEST
                 shutil.move(file_path, os.path.join(package_folder, file_name))
@@ -342,18 +347,25 @@ def _call_remote(self, remote, method, *args, **kwargs):
             raise ConanException(exc, remote=remote)
 
 
-def uncompress_file(src_path, dest_folder, scope=None):
-    if sys.version_info.minor < 14 and src_path.endswith("zst"):
+def uncompress_file(src_path, dest_folder, scope=None, extract_plugin=None):
+    if sys.version_info.minor < 14 and src_path.endswith("zst") and extract_plugin is None:
         raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', "
                              f"unsupported for Python<3.14 ")
+
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB
         if big_file:
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
+
+        if extract_plugin is not None:
+            if extract_plugin(src_path, dest_folder, scope) is not False:
+                # If the plugin returns false, fallback to Conan extraction
+                return
+
         with open(src_path, mode='rb') as file_handler:
-            tar_extract(file_handler, dest_folder)
+            tar_extract(fileobj=file_handler, destination_dir=dest_folder)
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

test/integration/command/cache/test_cache_save_restore.py

@@ -158,9 +158,13 @@ def test_cache_save_excluded_folders():
 
     # exclude source
     c.run("cache save * --no-source")
+    # Check default compression function is being used and not compression.py plugin one
+    assert "Compressing conan_cache_save.tgz\n" in c.out
     c3 = TestClient()
     shutil.copy2(cache_path, c3.current_folder)
     c3.run("cache restore conan_cache_save.tgz")
+    # Default decompress does not have any output
+    assert "Decompressing conan_cache_save.tgz" not in c3.out
     ref_layout = c3.get_latest_ref_layout(ref)
     assert not os.path.exists(os.path.join(ref_layout.source(), "mysrc.c"))