diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 94297504359..60af54abf90 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -135,6 +135,25 @@ def __init__(self, conan_api): self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) self._settings_yml = None self._remote_manager = None + self._compression_plugin = None + + @property + def compression_plugin(self): + # NOTE: Users cannot store a reference to this plugin, otherwise it will not + # be updated after a "conan config install/install-pkg" + if self._compression_plugin is None: + compression_plugin_path = HomePaths( + self._conan_api.home_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + self._compression_plugin = False # Avoid FS re-check + return None + mod, _ = load_python_file(compression_plugin_path) + # A plugin can provide just 1 of them + if not hasattr(mod, "tar_extract") and not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain " + "required `tar_extract` or `tar_compress` functions") + self._compression_plugin = mod + return self._compression_plugin def set_core_confs(self, core_confs): confs = ConfDefinition() @@ -165,6 +184,7 @@ def reinit(self): self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) self._remote_manager = None self._editable_packages = EditablePackages(self._conan_api.home_folder) + self._compression_plugin = None @property def settings_yml(self): @@ -180,7 +200,8 @@ def remote_manager(self): requester = self._conan_api._api_helpers.requester # noqa auth_manager = ConanApiAuthManager(requester, self._conan_api.home_folder, localdb, self.global_conf) - self._remote_manager = RemoteManager(self.cache, auth_manager, home_folder) + self._remote_manager = RemoteManager(self.cache, auth_manager, home_folder, + self.compression_plugin) return self._remote_manager @property diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index c95796ce4f0..cadb1e5f398 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -194,9 +194,10 @@ def sign(self, package_list): "https://docs.conan.io/2/reference/extensions/package_signing.html.") loader = self._api_helpers.loader + compress_plugin = self._api_helpers.compression_plugin preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, - self._api_helpers.global_conf) + self._api_helpers.global_conf, compress_plugin) # Some packages can have missing sources/exports_sources enabled_remotes = self._conan_api.remotes.list() preparator.prepare(package_list, enabled_remotes, None, force=True) @@ -373,7 +374,11 @@ def save(self, package_list: PackagesList, path, no_source=False) -> None: pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, tgz_name, os.path.dirname(path), compresslevel, recursive=True) + + plugin = self._conan_api._api_helpers.compression_plugin # noqa + compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None + compress_files(tar_files, tgz_name, os.path.dirname(path), compresslevel, + recursive=True, compress_plugin=compress_plugin) remove(pkglist_path) ConanOutput().success(f"Created cache save file: {path}") @@ -391,13 +396,26 @@ def restore(self, path) -> PackagesList: cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - with open(path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - fileobj = the_tar.extractfile("pkglist.json") - pkglist = fileobj.read() - the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) - the_tar.extractall(path=cache_folder) - the_tar.close() + plugin = self._conan_api._api_helpers.compression_plugin # noqa + extract_plugin = getattr(plugin, "tar_extract", None) if plugin else None + extracted = False + if extract_plugin: + extracted = extract_plugin(path, cache_folder, scope="Restore") + # If the plugin returns false, fallback to Conan extraction + + if extracted is False: + with open(path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) + the_tar.extractall(path=cache_folder) + the_tar.close() + + # Retrieve the package list from the already extracted archive + pkglist_path = os.path.join(cache_folder, "pkglist.json") + with open(pkglist_path) as file_handler: + pkglist = file_handler.read() + # Delete the pkglist.json file to keep cache clean + remove(pkglist_path) # After unzipping the files, we need to update the DB that references these files out = ConanOutput() diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index c18d532449d..fb52997f844 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -64,9 +64,10 @@ def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], raise ConanException("Empty string and patterns can not be mixed for metadata.") loader = self._api_helpers.loader + compress_plugin = self._api_helpers.compression_plugin preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, - self._api_helpers.global_conf) + self._api_helpers.global_conf, compress_plugin) preparator.prepare(package_list, enabled_remotes, metadata) signer = PkgSignaturesPlugin(self._api_helpers.cache, self._conan_api.home_folder) if signer.is_sign_configured: diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 4f46738a2dd..db64f2a961a 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -103,7 +103,7 @@ def get_compress_level(compressformat, global_conf): class PackagePreparator: - def __init__(self, loader, cache, remote_manager, global_conf): + def __init__(self, loader, cache, remote_manager, global_conf, compress_plugin): self._loader = loader self._remote_manager = remote_manager self._cache = cache @@ -113,6 +113,8 @@ def __init__(self, loader, cache, remote_manager, global_conf): compresslevel = get_compress_level(compressformat, global_conf) self._compressformat = compressformat self._compresslevel = compresslevel + plugin = compress_plugin # noqa + self._compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None def prepare(self, pkg_list, enabled_remotes, metadata, force=False): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) @@ -243,7 +245,8 @@ def _compressed_file(self, filename, files, download_folder, ref): file_name = filename + self._compressformat package_file = os.path.join(download_folder, file_name) compressed_path = compress_files(files, file_name, download_folder, - compresslevel=self._compresslevel, scope=str(ref)) + compresslevel=self._compresslevel, scope=str(ref), + compress_plugin=self._compress_plugin) assert compressed_path == package_file assert os.path.exists(package_file) return file_name @@ -336,13 +339,22 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False): +def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False, + compress_plugin=None): t1 = time.time() tgz_path = os.path.join(dest_dir, name) out = ConanOutput(scope=scope) out.info(f"Compressing {name}") + if compress_plugin is not None: + out.info(f"Compressing {name} using compression plugin") + compressed = compress_plugin(archive_path=tgz_path, files=files, recursive=recursive, + compresslevel=compresslevel, scope=scope) + if compressed is not False: + out.debug(f"{name} compressed in {time.time() - t1} time") + return tgz_path + if name.endswith("zst"): with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar: # noqa Py314 only for filename, abs_path in sorted(files.items()): diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index b3385c55d9a..2859f0ae67d 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -85,3 +85,7 @@ def settings_path_user(self): @property def config_version_path(self): return os.path.join(self._home, "config_version.json") + + @property + def compression_plugin_path(self): + return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py") diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index fce3be27d2c..1606cf970d8 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -28,11 +28,13 @@ class RemoteManager: _ErrorMsg = namedtuple("ErrorMsg", ["message"]) - def __init__(self, cache, auth_manager, home_folder): + def __init__(self, cache, auth_manager, home_folder, compression_plugin=None): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder + self._extract_plugin = getattr(compression_plugin, "tar_extract", None) \ + if compression_plugin else None def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: @@ -99,7 +101,8 @@ def _download_recipe(self, layout, ref, remote, metadata): tgz_file = zipped_files.pop(export_file, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref)) + uncompress_file(tgz_file, export_folder, scope=str(ref), + extract_plugin=self._extract_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -136,7 +139,8 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, layout.metadata(), files=zipped_files) # Only 1 file is guaranteed tgz_file = next(iter(zipped_files.values())) - uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), + extract_plugin=self._extract_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -191,7 +195,8 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(package_file) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref)) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), + extract_plugin=self._extract_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -342,18 +347,25 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None): - if sys.version_info.minor < 14 and src_path.endswith("zst"): +def uncompress_file(src_path, dest_folder, scope=None, extract_plugin=None): + if sys.version_info.minor < 14 and src_path.endswith("zst") and extract_plugin is None: raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', " f"unsupported for Python<3.14 ") + try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") + + if extract_plugin is not None: + if extract_plugin(src_path, dest_folder, scope) is not False: + # If the plugin returns false, fallback to Conan extraction + return + with open(src_path, mode='rb') as file_handler: - tar_extract(file_handler, dest_folder) + tar_extract(fileobj=file_handler, destination_dir=dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index 6f301daa87f..edcc5d59cec 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -158,9 +158,13 @@ def test_cache_save_excluded_folders(): # exclude source c.run("cache save * --no-source") + # Check default compression function is being used and not compression.py plugin one + assert "Compressing conan_cache_save.tgz\n" in c.out c3 = TestClient() shutil.copy2(cache_path, c3.current_folder) c3.run("cache restore conan_cache_save.tgz") + # Default decompress does not have any output + assert "Decompressing conan_cache_save.tgz" not in c3.out ref_layout = c3.get_latest_ref_layout(ref) assert not os.path.exists(os.path.join(ref_layout.source(), "mysrc.c")) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py new file mode 100644 index 00000000000..c7c1bd96d21 --- /dev/null +++ b/test/integration/extensions/test_compression_plugin.py @@ -0,0 +1,176 @@ +import os +import textwrap + + +from conan.internal.util.files import load +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +def test_compression_plugin_not_valid(): + """Test an error is raised if the compression plugin is not valid""" + c = TestClient(light=True) + c.save_home({"extensions/plugins/compression.py": ""}) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .", assert_error=True) + assert "ERROR: The 'compression.py' plugin does not contain" in c.out + + +def test_compression_plugin_fallbacks(): + """If the plugin methods returns False, fallback to Conan behavior""" + + c = TestClient(default_server_user=True, light=True) + compression_plugin = textwrap.dedent("""\ + import os + from conan.api.output import ConanOutput + + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Falling back to compress {name} with Conan!") + return False + + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Falling back to extract {name} with Conan!") + return False + """) + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt") + .with_package_file("some.lib", "lib"), + "myexportfile.txt": "something"}) + + c.run("create .") + c.run("upload * -r=default -c") + assert "pkg/1.0: Falling back to compress conan_export.tgz with Conan!" in c.out + assert "Falling back to compress conan_package.tgz with Conan!" in c.out + c.run("remove * -c") + c.run("install --requires=pkg/1.0") + assert "pkg/1.0: Falling back to extract conan_export.tgz with Conan!" in c.out + assert "pkg/1.0: Falling back to extract conan_package.tgz with Conan!" in c.out + + # same for cache save/restore + c.run("cache save *:*") + assert "Falling back to compress conan_cache_save.tgz with Conan!" in c.out + + c.run("remove * -c") + c.run("cache restore conan_cache_save.tgz") + assert "Restore: Falling back to extract conan_cache_save.tgz with Conan!" in c.out + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something" + c.run(f"cache path pkg/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709") + path = str(c.stdout).strip() + assert load(os.path.join(path, "some.lib")) == "lib" + + +def test_compression_plugin_correctly_load(): + """Test that the compression plugin is correctly loaded and used on: + - cache save/restore + - remote upload/download + """ + c = TestClient(default_server_user=True, light=True) + + compression_plugin = textwrap.dedent("""\ + import os, tarfile + from conan.api.output import ConanOutput + + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): + name = os.path.basename(archive_path) + assert name.endswith("xz") + ConanOutput(scope=scope).info(f"Compressing {name} with my Plugin!") + with tarfile.open(archive_path, f"w:xz", preset=compresslevel, + format=tarfile.PAX_FORMAT) as tgz: + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, filename, recursive=recursive) + + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Extracting {name} with my Plugin!") + with open(archive_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + the_tar.extractall(path=dest_dir) + the_tar.close() + """) + + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt") + .with_package_file("some.lib", "lib"), + "myexportfile.txt": "something"}) + + c.run("create .") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz") + assert "pkg/1.0: Compressing conan_export.txz with my Plugin!" in c.out + assert "Compressing conan_package.txz with my Plugin!" in c.out + + c.run("remove * -c") + c.run("install --requires=pkg/1.0") + assert "pkg/1.0: Extracting conan_export.txz with my Plugin!" in c.out + assert "pkg/1.0: Extracting conan_package.txz with my Plugin!" in c.out + + # same for cache save/restore + c.run("cache save *:* --file=save.txz") + assert "Compressing save.txz with my Plugin!" in c.out + + c.run("remove * -c") + c.run("cache restore save.txz") + assert "Restore: Extracting save.txz with my Plugin!" in c.out + + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something" + c.run(f"cache path pkg/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709") + path = str(c.stdout).strip() + assert load(os.path.join(path, "some.lib")) == "lib" + + +def test_compress_in_subdirectory(): + # https://github.com/conan-io/conan/issues/18259 + c = TestClient(light=True) + + compression_plugin = textwrap.dedent("""\ + import os, tarfile + from conan.api.output import ConanOutput + + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): + name = os.path.basename(archive_path) + assert name.endswith("xz") + ConanOutput(scope=scope).info(f"Compressing {name} with my Plugin!") + with tarfile.open(archive_path, f"w:xz", preset=compresslevel, + format=tarfile.PAX_FORMAT) as tgz: + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) + + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Extracting {name} with my Plugin!") + with open(archive_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + + for member in the_tar.getmembers(): + if member.name.startswith("conan/"): + member.name = member.name[len("conan/"):] # Strip 'conan/' prefix + the_tar.extract(member, path=dest_dir) + the_tar.close() + """) + + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt"), + "myexportfile.txt": "something"}) + + c.run("create .") + + c.run("cache save *:* --file=save.txz") + assert "Compressing save.txz with my Plugin!" in c.out + + c.run("remove * -c") + c.run("cache restore save.txz") + assert "Restore: Extracting save.txz with my Plugin!" in c.out + + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something"